ai-research-survey

scan.json (31341B)

---

 {
   "paper": {
     "title": "On Simulation-Guided LLM-based Code Generation for Safe Autonomous Driving Software",
     "authors": [
       "Ali Nouri",
       "Johan Andersson",
       "Kailash De Jesus Hornig",
       "Zhennan Fei",
       "Emil Knabe",
       "Håkan Sivencrona",
       "Beatriz Cabrero-Daniel",
       "Christian Berger"
     ],
     "year": 2025,
     "venue": "EASE 2025 (International Conference on Evaluation & Assessment in Software Engineering)",
     "arxiv_id": "2504.02141",
     "doi": "10.1145/3756681.3756987"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Zenodo links are provided for the interview protocol (doi:10.5281/zenodo.14783284) and sample generated codes/reports (doi:10.5281/zenodo.14783374), but the pipeline source code itself is not released. Only supplementary materials are available."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Sample generated codes and reports for one iteration (t13) are on Zenodo, and a YouTube demo is available. However, the full experimental data (all 40 code versions, all test reports, simulation scenarios, interview transcripts) is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Python, esmini simulator, and specific Ollama model IDs in footnote 2 (e.g., deepseek-coder:33b ID: acec7c0b0fd9), but provides no requirements.txt, Dockerfile, or detailed environment setup listing library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The pipeline architecture is described conceptually but not with enough operational detail (commands, configuration files) to reproduce."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw counts (e.g., 6 out of 20 successful for ACC, 1 out of 20 for CAEM) and percentages with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares models on success rates (e.g., GPT-4 vs Mistral:7b vs CodeGemma:7b) without any statistical significance tests. Claims of superiority are based solely on comparing raw counts."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw percentages (e.g., '9.2% improvement' for corrections, success rate from 30% to 35% for ACC) but no formal effect sizes (Cohen's d, odds ratios, etc.). The improvement percentages are raw differences without statistical context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "N=20 pipeline initiations and N=11 interviewees. No justification for why 20 initiations were chosen, no power analysis, and no discussion of whether these sample sizes are adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across runs. Each of the 20 initiations is shown individually in Fig 4 but no aggregate variability measures are computed."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple LLM models are compared: GPT-4, Codellama:34b, DeepSeek-r1:32b, DeepSeek-Coder:33b, CodeGemma:7b, and Mistral:7b (Fig 5). Initial code generation vs correction-loop versions are also compared."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models used (GPT-4, DeepSeek, CodeGemma, Mistral, Codellama) were contemporary at the time of the study. The paper was submitted January 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The comparison between initial code generation (Specification Prompt only) vs the correction loop (with simulation feedback) effectively ablates the feedback mechanism. Results show 9.2% average improvement with the correction loop (Sec 7.1)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple evaluation criteria are used: compilability (syntax errors), number of test cases passed (TC1-TC7), expert ratings (useful/very useful), and qualitative analysis of failure modes."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "An interview study with 11 experts from 2 OEMs evaluated the prototype (Sec 7.3). Experts watched a pre-recorded demo and rated usefulness, strengths, weaknesses, and proposed improvements."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The same 7 test cases (TC1-TC7) used to provide feedback to the LLM during the correction loop are also used as the final evaluation criteria. There is no separate held-out evaluation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (Fig 5), per function (ACC vs CAEM), per iteration (Fig 4), and by outcome type (non-compilable, failed, corrected, successful)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases are discussed in detail: 3 instances of non-executable code, 3 runtime errors (Sec 7.1), regression during correction (Fig 4), complete failure of open-source models on CAEM (Sec 7.2), and token limitation effects on correction loops."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results: all open-source models failed for CAEM, correction loop sometimes causes regression, larger models don't necessarily outperform smaller ones (Mistral:7b > Codellama:34b), token limitations prevented open-source model corrections."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: they developed and evaluated a prototype (Sec 6-7), report experimental results for multiple models (Fig 5), and assessed with 11 experts (Sec 7.3). No overclaiming in the abstract."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim — that the feedback loop improves code quality — is supported by a controlled comparison: same model, same prompts, with vs without correction. The 9.2% improvement and specific examples (C25→C26 in t13) demonstrate the mechanism."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Sec 7.5 (External Validity) explicitly bounds the results: 'designed with specific components such as GPT-4 for code generation, esmini for simulation, and for generating Python code for ADS.' The paper acknowledges adaptability claims are aspirational."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Sec 7.5 discusses alternatives: LLM stochastic behavior causing variability, potential data leakage for ACC vs CAEM, simulation model inaccuracy. Sec 7.4 discusses token limitations as explanation for open-source model failures in correction loops."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures pass/fail on 7 simulated test scenarios and frames this as 'safety evaluation' and 'safety-related KPIs.' The gap between passing 7 predefined scenarios in esmini and actual safety of ADS code is not explicitly discussed. The paper acknowledges simulation limitations but does not distinguish the proxy (scenario pass rate) from the claimed outcome (safety)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Open-source models have Ollama hash IDs in footnote 2 (e.g., deepseek-coder:33b ID: acec7c0b0fd9), but GPT-4 — the primary model — is specified only as 'GPT-4' without a version, snapshot date, or API version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The Correction Prompt template is shown in Fig 2 with structural placeholders. The Specification Prompt structure is described (Context + Task Description) in Sec 6.1. But the actual prompt text is not provided — only descriptions of what each section contains."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other API/inference parameters are reported for any model. These significantly affect LLM output quality and reproducibility."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline architecture is well-described: LLM code generation → esmini simulation → rule-based report generator → natural language feedback → correction prompt (Fig 2). The baseline selection strategy, fresh-start mechanism, and iteration logic are documented (Fig 3, Sec 6.2)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: simulation generates tabular logs of positions/velocities → report generator translates to natural language → pass/fail assessment against acceptance criteria (Sec 6.2). Test case specifications are detailed in Sec 4.4."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Sec 7.5 'Threats to Validity' provides substantive discussion organized into Construction Validity, Internal Validity, and External Validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific to this study: simulation model inaccuracy for ADS (construction), LLM stochastic behavior mitigated by 20 initiations (internal), CAEM chosen to minimize data leakage (internal), specific components limiting generalizability (external)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope: prototype is for 'concept phase and function abstraction level' (Sec 4.1), not production code. External validity (Sec 7.5) states the specific components. The disclaimer notes the work is 'not used in any engineering of production related projects.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Only a sample of generated codes and reports for iteration t13 is available via Zenodo (doi:10.5281/zenodo.14783374). The full set of 40 code versions, test reports, and simulation logs is not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The experiment procedure is clearly described: 20 pipeline initiations per model per function, 2 versions per initiation (initial + correction), evaluation against 7 test cases, expert interview protocol (Sec 3.3-3.4, Sec 7.1)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Interview participants: '11 experts selected from 2 OEMs,' minimum 5 years ADS experience, average 12 years, roles listed (manager, product owners, V&V engineers, etc.), from 3 countries (Sec 3.4). Selection criteria are stated."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: 20 initiations → 40 versions → 6 non-executable filtered out → 14 initial + 14 enhanced versions analyzed (Sec 7.1). Filtering at each stage is explained with counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section: 'partially supported by Sweden's Innovation Agency (Vinnova, diarienummer: 2021-02585), and by the Wallenberg AI Autonomous Systems and Software Program (WASP) funded by the Knut and Alice Wallenberg Foundation.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: 5 authors from Volvo Cars and/or Chalmers University, 2 from University of Gothenburg. Volvo Cars is identified as the case company."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are Vinnova (Swedish government innovation agency) and WASP (academic foundation). Neither has a financial stake in the performance of specific LLM models or the pipeline."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included. Authors from Volvo Cars are evaluating a prototype relevant to Volvo's business, but this potential conflict is not explicitly declared beyond affiliation listing."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for any of the models used (GPT-4, Codellama, DeepSeek, CodeGemma, Mistral). The paper discusses leakage conceptually but not with temporal specifics."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Sec 4.2 explicitly discusses this: 'AEB and ACC are mature, there is plenty of data in public databases, which increases the probability of the LLM being trained on them, posing a threat to validity.' They chose CAEM specifically because it is 'immature with respect to existing publicly available data.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Sec 2 discusses benchmark leakage risk. Sec 4.2 addresses it by selecting CAEM as a novel function to minimize leakage. Sec 7.4 interprets the ACC vs CAEM performance gap partly through this lens: 'models are specifically optimised to pass these known tasks and benchmarks rather than to solve truly novel tasks.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the interview study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The paper states that 'ethical guidelines for software engineering interview studies, such as obtaining consent, anonymization, and confidentiality, as proposed by Strandberg [27] were followed,' but no IRB or ethics board approval is mentioned."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Participant demographics are reported: roles (manager, product owners, V&V engineers, data scientist, system architect, programmer, system safety experts), average 12 years experience, 3 countries, 2 OEMs (Sec 3.4)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Sec 3.4: 'To be eligible, the participants had to have at least 5 years of experience in ADS development.'"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative interview study, not an experimental study with treatment/control conditions. Randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative interview study where all participants view the same demo. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "Sec 3.4 mentions '12 experts from two OEMs' but the Participants subsection and all results refer to '11 experts.' This discrepancy (12 invited vs 11 participated?) is not explained."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Sec 7.4: 'Each full execution of the pipeline took less than 3 minutes from the initiation to delivering both the initial and enhanced versions, including simulations and report generation. This means that 20 full executions of the pipeline required less than an hour.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Only wall-clock time is reported. No hardware specifications (GPU, CPU, RAM), API costs, or total compute budget are stated. For open-source models running locally, the hardware used is not specified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No random seeds are set or reported. The 20 independent pipeline runs capture LLM stochasticity but this is not framed as seed sensitivity analysis, and no aggregate variability measures are computed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Sec 7.1: 'the pipeline was initiated 20 times, resulting in 40 versions of the code for the CAEM function using GPT-4.' Sec 7.2 states 20 codes per model per function."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Prompts were iteratively refined during design cycles but no systematic hyperparameter search is reported. No temperature, prompt variation, or configuration search budget is stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All 40 versions for GPT-4/CAEM are shown in Fig 4. All models' results are reported in Fig 5. The baseline selection strategy is transparent (best-performing code at each step, Sec 6.2, Fig 3). No cherry-picking — all configurations are reported."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so correction for multiple comparisons does not apply. The absence of statistical tests is already captured in significance_tests."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed and evaluate their own pipeline. The interview study partially mitigates this with external expert evaluation, but the authors do not acknowledge author-evaluation bias for the experimental results."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models range from 7b to 34b parameters plus GPT-4. The paper notes 'more parameters do not necessarily lead to better results' but does not compare performance at matched compute budgets or report per-model compute costs."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Sec 2 and 4.2 explicitly discuss construct validity of standard benchmarks: 'benchmark leakage creates the risk of assessing the model's memory rather than its ability to handle unseen tasks.' They propose novel evaluation tasks as an alternative."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All models are evaluated using the same pipeline, same prompts, same simulation environment, and same test cases. The scaffold is controlled across model comparisons. Differences in correction loop effectiveness are attributed to token limitations (Sec 7.4)."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Sec 4.2 discusses that ACC/AEB have 'plenty of data in public databases' likely in LLM training data, while CAEM is 'immature with respect to existing publicly available data.' The choice of CAEM as the primary evaluation function directly addresses temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline provides simulation feedback to the LLM during correction loops. While this is by design, the paper does not discuss whether this feedback provides information that would not be available in real deployment or whether it constitutes feature leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 20 pipeline runs per model are treated as independent samples but there is no discussion of whether they are truly independent (same model weights, similar stochastic patterns, shared prompt structure)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The study uses a concrete prevention method: selecting CAEM as a novel function with minimal public data to avoid training data contamination, then comparing performance against ACC (known function with more public data) to detect leakage effects (Sec 4.2, 7.4)."
       }
     }
   },
   "claims": [
     {
       "claim": "The simulation-based feedback loop improves generated code, with an average 9.2% improvement in passed test cases across all corrections and 37% considering only successful improvements.",
       "evidence": "Sec 7.1: 14 initial generations and 14 enhanced versions analyzed. 5 of 14 corrections showed improvement. For ACC, success rate increased from 30% (6/20) to 35% (5/14). The gold baseline (all 7 TCs passed) was achieved through correction in iteration t13.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 significantly outperforms all tested open-source models, being the only model to produce successful code for both ACC (11 successful) and CAEM (2 successful).",
       "evidence": "Fig 5 and Sec 7.2: GPT-4 produced 11 successful codes for ACC and 2 for CAEM. Open-source models produced at most 3 (Mistral for ACC) and 0 for CAEM. All GPT-4 generated code compiled, unlike open-source models with many syntax errors.",
       "supported": "strong"
     },
     {
       "claim": "More parameters in an LLM do not necessarily lead to better code generation: Mistral:7B and CodeGemma:7B outperform CodeLlama:34B and DeepSeek-r1:32B.",
       "evidence": "Fig 5: Mistral:7b produced 3 successful ACC codes, CodeGemma:7b produced 1, while CodeLlama:34b and DeepSeek models produced 0. CodeLlama had the highest non-compilable count (26).",
       "supported": "moderate"
     },
     {
       "claim": "Standard benchmarks may overstate model performance due to benchmark leakage — models perform much worse on novel tasks.",
       "evidence": "Sec 4.2 and 7.4: ACC (mature, more public data) had much higher success rates than CAEM (novel function). Open-source models succeeded on ACC but completely failed on CAEM. However, this is confounded with task complexity.",
       "supported": "weak"
     },
     {
       "claim": "The prototype is useful for industrial ADS development, particularly for prototyping and concept-phase work.",
       "evidence": "Sec 7.3: 7/11 experts rated 'very useful,' 4 rated 'useful.' Cited benefits include reduced workload (8 experts), prototyping aid (3 experts), automation (3 experts), and visualization (1 expert).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study", "qualitative"],
   "key_findings": "This paper develops and evaluates a simulation-guided LLM code generation pipeline for autonomous driving software (ACC and CAEM functions). GPT-4 is the only model that successfully generates code for the novel CAEM function, while open-source models (Codellama:34b, DeepSeek, CodeGemma:7b, Mistral:7b) largely fail. The simulation-based correction feedback loop provides a modest 9.2% average improvement in passed safety test cases. An interview study with 11 industry experts validates the tool's usefulness for concept-phase prototyping, while emphasizing the need for human oversight before production deployment.",
   "red_flags": [
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "The paper compares 6 models across 2 functions making claims of superiority (GPT-4 best, smaller models outperform larger ones) based solely on raw counts from N=20 runs with no significance tests, confidence intervals, or variance measures."
     },
     {
       "flag": "Small sample sizes",
       "detail": "N=20 pipeline initiations per model per function and N=11 interviewees. With high LLM output variability and only 2 successful CAEM codes from GPT-4 (out of 40 versions), the sample is too small for robust conclusions about model capabilities."
     },
     {
       "flag": "Confounded comparison between ACC and CAEM",
       "detail": "The claim that benchmark leakage explains the ACC vs CAEM performance gap is confounded with task complexity — CAEM is explicitly described as more complex than ACC (multi-agent evasive maneuvers vs cruise control). The paper cannot separate leakage from complexity effects."
     },
     {
       "flag": "Potential social desirability bias in expert interviews",
       "detail": "Volvo Cars authors interview experts at their own company and a supplier company about a tool they developed. All 11 experts rated the tool as useful or very useful, which may reflect social desirability rather than genuine assessment."
     },
     {
       "flag": "Unexplained participant count discrepancy",
       "detail": "Sec 3.4 states '12 experts from two OEMs are interviewed' but the Participants subsection and all subsequent analysis refer to 11 experts. The discrepancy is not explained (dropout? pilot participant?)."
     }
   ],
   "cited_papers": [
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming – The Rise of Code Intelligence",
       "authors": ["D. Guo", "Q. Zhu", "D. Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "LLM code generation capability evaluation, one of the models tested in this study."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "relevance": "Foundational Codex/HumanEval paper on LLM code generation benchmarking and IP/security concerns."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Rigorous evaluation of LLM code correctness, relevant to methodology quality in code generation studies."
     },
     {
       "title": "An Empirical Study of the Code Generation of Safety-Critical Software Using LLMs",
       "authors": ["M. Liu", "J. Wang", "T. Lin"],
       "year": 2024,
       "relevance": "Directly relevant prior work on LLM code generation for safety-critical automotive software with prompt engineering techniques."
     },
     {
       "title": "Fully autonomous programming with large language models",
       "authors": ["V. Liventsev", "A. Grishina", "A. Härmä", "L. Moonen"],
       "year": 2023,
       "relevance": "Iterative Synthesis, Execution, and Debugging (SED) approach for LLM programming, key precursor to the correction loop concept."
     },
     {
       "title": "Don't make your LLM an evaluation benchmark cheater",
       "authors": ["K. Zhou", "Y. Zhu", "Z. Chen"],
       "year": 2023,
       "arxiv_id": "2311.01964",
       "relevance": "Benchmark leakage in LLM evaluation, directly cited to motivate the novel-task evaluation approach."
     },
     {
       "title": "An Empirical Evaluation of GitHub Copilot's Code Suggestions",
       "authors": ["N. Nguyen", "S. Nadi"],
       "year": 2022,
       "relevance": "Empirical evaluation of AI code generation quality across programming languages, provides the Python baseline (42%) cited in this paper."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig"],
       "year": 2024,
       "relevance": "Major LLM code generation benchmark, cited regarding readability and comprehensiveness of LLM-generated code."
     },
     {
       "title": "A Large-Scale Survey on the Usability of AI Programming Assistants: Successes and Challenges",
       "authors": ["J. T. Liang", "C. Yang", "B. A. Myers"],
       "year": 2023,
       "arxiv_id": "2303.17125",
       "relevance": "Large-scale survey on AI programming assistant usability, cited for LLM code generation limitations and IP concerns."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["N. Shinn", "F. Cassano", "E. Berman"],
       "year": 2023,
       "arxiv_id": "2303.11366",
       "relevance": "Key prior work on iterative LLM self-improvement through verbal feedback, foundational to the correction loop concept."
     },
     {
       "title": "Langprop: A code optimization framework using large language models applied to driving",
       "authors": ["S. Ishida", "G. Corrado", "G. Fedoseev"],
       "year": 2024,
       "relevance": "LLM code optimization applied specifically to driving tasks, most closely related prior work in the autonomous driving domain."
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "authors": ["E. Nijkamp", "B. Pang", "H. Hayashi"],
       "year": 2023,
       "arxiv_id": "2203.13474",
       "relevance": "Multi-turn code generation and task decomposition approach for LLM programming."
     }
   ]
 }