ai-research-survey

scan.json (29210B)

---

 {
   "paper": {
     "title": "Reasoning Runtime Behavior of a Program with LLM: How Far Are We?",
     "authors": ["Junkai Chen", "Zhiyuan Pan", "Xing Hu", "Zhenhao Li", "Ge Li", "Xin Xia"],
     "year": 2024,
     "venue": "International Conference on Software Engineering (ICSE)",
     "arxiv_id": "2403.16437",
     "doi": "10.1109/ICSE55347.2025.00012"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "REval evaluates 15 LLMs on four code reasoning tasks (code coverage, program state, execution path, output prediction) and a novel Incremental Consistency metric. Most models show poor performance (average 44.4% accuracy, 10.3 IC score). GPT-4-Turbo dominates with 75.0% accuracy and 42.5 IC score, more than doubling the second-best IC score. Execution Path Prediction is the hardest task, with most models below 10% accuracy. Chain-of-Thought prompting helps intermediate reasoning tasks but hurts output prediction.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a project page at https://r-eval.github.io and a replication package at figshare (https://figshare.com/s/e5de95bd79ab5ddea76c). The abstract states 'Our code, data and REval leaderboard are available.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The adapted benchmark data is available via the project page and figshare replication package. The base benchmarks (HumanEval, ClassEval) are publicly available on HuggingFace."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'a Linux server with 8 NVIDIA A800 GPUs' and deployment via vLLM, but does not provide a requirements.txt, Dockerfile, or specific library versions needed to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions a replication package with 'detailed information, such as model IDs and URLs' but does not include step-by-step reproduction instructions within the paper itself. No 'Reproducing Results' section or runnable scripts are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Results for all open-source models are reported as 'mean±standard deviation' across 5 repetitions (Table III). For example, CodeLlama-7B-Base CCP: 54.3±0.5."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used despite numerous comparative claims (e.g., 'outperforms the second best by a large margin'). All comparisons are based on raw numerical differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute differences are reported with baseline context throughout, e.g., 'an absolute improvement of 19.3%' (GPT-4-Turbo vs GPT-3.5), 'a relative improve by over 100% (i.e., 14.4% → 29.2%)', providing enough context to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 3,152 problems (Table I). No justification is given for why this size is sufficient, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported across 5 experimental repetitions for all open-source models in Table III (e.g., '54.3±0.5'). GPT models lack variance due to budget constraints, which is explicitly acknowledged."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "15 models are compared against each other across multiple tasks. Both general-purpose and code-specialized LLMs serve as mutual baselines. The paper also compares against prior work CRUXEval's scope."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models include GPT-4-Turbo, StarCoder2 (02/2024), Gemma (02/2024), and Mistral-7B (01/2024), all contemporary at time of study. Some older models like CodeLlama (08/2023) are included for comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study isolating contributions of individual framework components. The CoT vs few-shot comparison (last row of Table III) tests prompting strategy on a single model but does not ablate framework design choices."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Accuracy and F1 for CCP, Accuracy for PSP/EPP/OP, Average Accuracy across tasks, and the novel IC Score. Table III reports all of these."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed. All evaluation is automated using ground truth from program execution traces. No human assessment of benchmark quality or model output quality is reported."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The entire adapted benchmark serves as the test set. Models are evaluated with few-shot prompting where demonstrations are separate from the evaluated problems. No training or tuning is done on this data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task (CCP, PSP, EPP, OP) for each model in Table III, revealing that EPP is much harder than OP. Per-model breakdowns across all dimensions are provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-A presents a case study from EPP (HumanEval/59) showing where GPT-3.5 and CodeLlama-34B fail but GPT-4-Turbo succeeds, with analysis of why models choose incorrect answers."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: CoT prompting decreases OP accuracy by 6.8% (Section V-A); Magicoder fine-tuning improves code generation but not reasoning; code LLMs don't significantly outperform general LLMs of the same size."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'average accuracy of 44.4%' and 'average IC score of 10.3' are directly supported by Table III, which shows these exact averages across all 15 models."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'additional training with Python corpora leads to an improvement' and 'instruction tuning techniques brings gains' by comparing model variants, but confounds (different training data, architectures) are not controlled for. These are observational comparisons presented with causal language."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'How Far Are We?' implies a general assessment of the field, but results are limited to 2 Python benchmarks (HumanEval and ClassEval) with 15 specific models. The threats-to-validity section acknowledges benchmark limitations but the title and framing overgeneralize."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper speculates about causes (e.g., 'may demonstrate that apart from parameter size, the model architecture and training strategy also play an important role') but does not systematically consider confounds or alternative explanations. The threats section discusses methodological limitations but not alternative interpretations of the observed results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper equates performance on four synthetic tasks (CCP, PSP, EPP, OP) with 'code reasoning ability' without discussing whether these tasks fully capture what code reasoning entails. No discussion of the gap between benchmark performance and actual program comprehension capability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Model names are given (e.g., 'GPT-3.5-Turbo', 'GPT-4-Turbo') but without specific API versions or snapshot dates. Table II lists release dates (e.g., '01/2024') but not API version strings like 'gpt-4-turbo-2024-01-25'. Full details are deferred to the replication package."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Fig. 3 shows a prompt template with placeholders (e.g., '[PYTHON]...', '[QUESTION]...', '[ANSWER]...'). The actual few-shot demonstration content and system message text are not provided in the paper. Full prompts are deferred to the replication package."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-D reports temperature=0.8, max tokens=256 (direct)/1024 (CoT), 5 repetitions for open-source models, and 'default settings in vLLM for the rest of the parameters.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are directly prompted via API/vLLM with standard few-shot or CoT prompting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-D describes the full benchmark construction pipeline: runtime behavior extraction via custom program tracer, then problem construction with specific filtering rules for each task (CCP/EPP: last statement in control flow blocks; PSP: assignment/return/changed variables with priority rules; OP: assertion masking)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-D 'Threats to Validity' provides a substantive discussion covering internal threats (selection criteria, task coverage) and external threats (model and benchmark generalizability)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: selection criteria for statements/variables may not represent runtime state; dynamic features like memory allocation and exception handling are not covered; results restricted to specific models and benchmarks. Mitigations are described for each."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'there are still some dynamic features such as memory allocation and exception handling which may help measure code models, and we have not explored yet' and 'the results are restricted to the specific collection of code models and base benchmarks.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is provided at figshare with code and data. The base benchmarks are publicly available on HuggingFace. Model outputs appear to be included based on the replication package reference."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-D describes the data collection process in detail: execution of canonical solutions from HumanEval and ClassEval using provided test cases with a customized program tracer, recording statement execution and local variable states at each step."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard public benchmarks (HumanEval and ClassEval)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the filtering rules are described in detail (Section III-D2), no intermediate counts are provided. The paper goes from base benchmarks to 3,152 final problems (Table I) without documenting how many candidates existed at each filtering stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This research is supported by the Ningbo Natural Science Foundation (No. 2023J292)' and computing resources from 'the Supercomputing Center of Hangzhou City University.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Zhejiang University, York University, and Peking University. None of the authors are affiliated with companies whose products are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Ningbo Natural Science Foundation is a public funding body with no financial interest in the relative performance of any evaluated LLM."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Table II lists model release dates (e.g., '08/2023', '02/2024') but not training data cutoff dates. Release date is not the same as training data cutoff."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval or ClassEval solutions appeared in any model's training data. HumanEval was published in 2021 and is widely available; all evaluated models were trained after 2021."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021 and has been publicly available on GitHub since then. All 15 evaluated models were released after 2021 and could have trained on HumanEval solutions. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'limited budget' as the reason for not repeating GPT experiments, but does not report actual inference costs, API costs, or per-example costs for any model."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (8 NVIDIA A800 GPUs) but total GPU hours, wall-clock time for experiments, or total API spend are not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Experiments for all open-source models are repeated 5 times with temperature=0.8, and mean±std are reported in Table III, capturing seed sensitivity. However, GPT models are not repeated due to budget."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-D explicitly states: 'experiments for all open-source models with few-shot prompting are repeated five times.' GPT models are run once, with the reason given."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is reported. Temperature=0.8 is adopted from prior work (CRUXEval) without exploring alternatives. No search budget is mentioned."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a fixed configuration following prior work (CRUXEval) rather than selecting a best configuration. All models use the same settings, and all results are reported — no cherry-picking of configurations."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many model-to-model comparisons across 4 tasks with 15 models but performs no statistical tests at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the REval benchmark and evaluate all models on it. No discussion of potential bias from designing both the evaluation framework and conducting the evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Model sizes range from 2B to unknown (GPT-4-Turbo), but performance is not analyzed as a function of compute. Closed-source models like GPT-4-Turbo likely use vastly more compute than 3B models but this difference is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether CCP, PSP, EPP, and OP actually measure 'code reasoning ability' as claimed. No analysis of construct validity or comparison with alternative code reasoning definitions."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models are prompted directly via API/vLLM."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021. All models were released 2023-2024 and could have been trained on HumanEval solutions. The paper does not discuss this temporal leakage risk."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the prompts or few-shot examples leak information about correct answers. The prompt structure (e.g., providing the full program and asking about specific lines) is not analyzed for information leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Multiple problems are generated from the same program (different statements/variables), creating non-independent test items. This structural non-independence is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, decontamination, or temporal splits are used."
       }
     }
   },
   "claims": [
     {
       "claim": "Most LLMs show unsatisfactory performance on Runtime Behavior Reasoning with an average accuracy of 44.4%.",
       "evidence": "Table III shows average accuracy across all 15 models on 4 tasks, computed as the mean of CCP (61.0%), PSP (35.6%), EPP (19.4%), and OP (61.8%).",
       "supported": "strong"
     },
     {
       "claim": "GPT-4-Turbo demonstrates clear superiority over all other models, achieving 75.0% average accuracy and 42.5 IC score.",
       "evidence": "Table III shows GPT-4-Turbo at 88.4% CCP, 71.4% PSP, 57.7% EPP, 82.6% OP, and IC score 42.5 — more than double the second-best IC score of 20.6 (GPT-3.5).",
       "supported": "strong"
     },
     {
       "claim": "Execution Path Prediction is the most challenging task, with most models below 10% accuracy.",
       "evidence": "Table III shows EPP average of 19.4%, with 7 of 15 models below 10% accuracy. Only GPT-4-Turbo (57.7%) and Mistral-7B (35.8%) exceed 30%.",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-Thought prompting improves intermediate reasoning tasks but hurts Output Prediction.",
       "evidence": "Last row of Table III: CodeLlama-7B-Instruct with CoT shows EPP improvement from 10.8% to 21.4% but OP decrease from 62.6% to 55.8% (absolute decrease of 6.8%).",
       "supported": "moderate"
     },
     {
       "claim": "Code LLMs do not exhibit an obviously leading advantage over general LLMs of the same size in code reasoning.",
       "evidence": "Table III shows general LLMs like Mistral-7B (48.0% avg) outperforming code LLMs of similar size like StarCoder2-7B (39.6%) and CodeLlama-7B-Instruct (38.5%).",
       "supported": "moderate"
     },
     {
       "claim": "There is a strong positive correlation between code reasoning and code generation (Pearson > 0.7).",
       "evidence": "Table IV reports Pearson correlation: RBR-HE = 0.772, IC-HE = 0.724. Both exceed the 0.7 threshold for strong positive correlation.",
       "supported": "moderate"
     },
     {
       "claim": "Larger model variants within the same family show better code reasoning performance.",
       "evidence": "CodeLlama-Instruct series: 7B→13B→34B shows EPP accuracy from 10.8%→14.4%→29.2%; StarCoder2: 3B→7B→15B shows average accuracy from 37.3%→39.6%→50.5%.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination analysis on well-known benchmarks",
       "detail": "HumanEval was published in 2021 and is widely available on GitHub. All 15 evaluated models were trained after 2021 and could have memorized solutions. The paper's adapted benchmark derives from HumanEval, so contamination could inflate code reasoning scores — yet this is never discussed."
     },
     {
       "flag": "No statistical significance tests for comparative claims",
       "detail": "The paper makes numerous claims like 'outperforms by a large margin' and 'does not exhibit an obviously leading advantage' based purely on numerical comparisons without any statistical testing. With 15 models, 4 tasks, and 5 repetitions, proper statistical analysis is feasible but absent."
     },
     {
       "flag": "GPT models not repeated",
       "detail": "Experiments for GPT-3.5-Turbo and GPT-4-Turbo are run only once due to 'limited budget,' while all comparative claims heavily feature these models. The two strongest-performing models lack any uncertainty quantification."
     },
     {
       "flag": "Non-independence of test problems",
       "detail": "Multiple problems are generated from the same program (different statements, variables), creating correlated test items. Accuracy metrics treat each problem as independent, potentially inflating confidence in results."
     }
   ],
   "cited_papers": [
     {
       "title": "CRUXEval: A Benchmark for Code Reasoning, Understanding and Execution",
       "authors": ["Alex Gu", "Baptiste Rozière", "Hugh Leather", "Armando Solar-Lezama", "Gabriel Synnaeve", "Sida I. Wang"],
       "year": 2024,
       "arxiv_id": "2401.03065",
       "relevance": "Directly related benchmark for evaluating LLM code reasoning via input/output prediction; REval extends this approach to runtime behavior."
     },
     {
       "title": "CodeMind: A Framework to Challenge Large Language Models for Code Reasoning",
       "authors": ["Changshu Liu", "Shizhuo Dylan Zhang", "Reyhaneh Jabbarvand"],
       "year": 2024,
       "arxiv_id": "2402.09664",
       "relevance": "Concurrent framework for evaluating code reasoning with LLMs including specification reasoning and execution prediction."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, the foundational code generation benchmark used as a base benchmark in REval."
     },
     {
       "title": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-Level Code Generation",
       "authors": ["Xueying Du", "Mingwei Liu", "Kaixin Wang"],
       "year": 2024,
       "relevance": "Class-level code generation benchmark used as the second base benchmark for REval, representing context-aware programming scenarios."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major open-source code LLM family evaluated in the study; demonstrates model variant effects on code reasoning."
     },
     {
       "title": "Magicoder: Source Code Is All You Need",
       "authors": ["Yuxiang Wei", "Zhe Wang", "Jiawei Liu", "Yifeng Ding", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2312.02120",
       "relevance": "Fine-tuned code LLM showing that code generation improvements don't necessarily transfer to code reasoning tasks."
     },
     {
       "title": "TRACED: Execution-Aware Pre-Training for Source Code",
       "authors": ["Yangruibo Ding", "Benjamin Steenhoek", "Kexin Pei", "Gail Kaiser", "Wei Le", "Baishakhi Ray"],
       "year": 2024,
       "relevance": "Execution-aware pre-training technique combining static and dynamic code characteristics, directly relevant to understanding runtime behavior learning."
     },
     {
       "title": "Beyond Accuracy: Evaluating Self-Consistency of Code LLMs",
       "authors": ["Marcus J. Min", "Yangruibo Ding", "Luca Buratti", "Saurabh Pujar", "Gail Kaiser", "Saikat Jana", "Baishakhi Ray"],
       "year": 2023,
       "relevance": "Evaluates self-consistency of code LLMs via back-translation, directly related to REval's incremental consistency metric."
     },
     {
       "title": "Unsupervised Evaluation of Code LLMs with Round-Trip Correctness",
       "authors": ["Miltiadis Allamanis", "Sheena Panthaplackel", "Pengcheng Yin"],
       "year": 2024,
       "arxiv_id": "2402.08699",
       "relevance": "Proposes round-trip correctness for unsupervised code LLM evaluation, a complementary approach to REval's supervised consistency evaluation."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational prompting technique evaluated in REval, showing mixed results for code reasoning tasks."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The Next Generation",
       "authors": ["Anton Lozhkov", "Raymond Li", "Loubna Ben Allal", "Federico Cassano"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Open-source code LLM series evaluated in the study, trained on The Stack v2 dataset with varied architectures."
     },
     {
       "title": "Code Simulation Challenges for Large Language Models",
       "authors": ["Emanuele La Malfa", "Christoph Weinhuber", "Orazio Torre", "Fangru Lin", "Anthony Cohn", "Nigel Shadbolt", "Michael Wooldridge"],
       "year": 2024,
       "arxiv_id": "2401.09074",
       "relevance": "Concurrent work on simulating code execution with LLMs, focusing on algorithm complexity; related but more limited than REval's runtime behavior approach."
     }
   ]
 }