ai-research-survey

scan.json (29888B)

---

 {
   "paper": {
     "title": "Enhancing LLM Code Generation: A Systematic Evaluation of Multi-Agent Collaboration and Runtime Debugging for Improved Accuracy, Reliability, and Latency",
     "authors": [
       "Nazmus Ashrafi",
       "Salah Bouktif",
       "Mohammed Mediani"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2505.02133",
     "doi": "10.48550/arXiv.2505.02133"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Debugging-based approaches generally outperform multi-agent agentic workflows for LLM code generation across 19 models on HumanEval/HumanEval+. Combining a simple two-agent (Analyst-Coder) workflow with debugging yields a modest 0.68% mean accuracy improvement over debugging alone, but adding a more complex three-agent system (Analyst-Coder-Tester) reduces accuracy for most models. The benefit of combining approaches is greatest when the performance gap between debugging and agentic methods is small for a given model. Reduced agentic complexity generally produces more rigorous code as measured by HumanEval+ score retention.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository is provided (https://github.com/nazmus-ashrafi/multiagent_vs_debugger) with agent prompts. Referenced in Section 3 and Section 4.3.1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses HumanEval and HumanEval+, both publicly available benchmark datasets. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup is provided. The paper does not describe library versions or dependencies needed to reproduce the experiments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper or referenced. The GitHub repo contains prompts but no described workflow for replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 2 are point estimates (pass@1 scores) with no confidence intervals or error bars. The t-tests report p-values but no CIs on the accuracy differences."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "One-tailed paired t-tests are used (Section 4.2) to compare ACT+Debug vs ACT alone and ACT+Debug vs Debug alone, with explicit hypothesis formulation and p-value reporting."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Mean accuracy differences are reported with baseline context: e.g., ACT+Debug mean 64.82% vs ACT alone 57.16% vs Debug alone 63.86% (Section 4.2). Per-model improvements are given in Table 2 with specific percentage point changes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 19 LLMs were selected as the sample size. No power analysis is discussed. The choice appears driven by availability rather than statistical planning."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly uses n=1 (one sample per problem) per Section 4.1.1: 'we chose to generate only one sample per problem (n=1).' No variance across runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six approaches are compared: Basic, AC, ACT, Debugger, AC+Debugger, and ACT+Debugger (Section 4.1.3, Figure 8). These include individual components and combinations."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include the LDB debugger (2024) and self-collaboration framework (2023), both recent at time of writing. Models tested include GPT-4o, Claude 3.5 Sonnet, DeepSeek-V3, all contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Section 4.3) systematically removes/adds components: Basic → AC → ACT → Debugger → AC+Debug → ACT+Debug, isolating the contribution of each component. Figure 8 shows the composition of each segment."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates pass@1 accuracy, code rigorousness (HumanEval vs HumanEval+ accuracy drop, RQ3), and generation latency (RQ4, Table 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via pass/fail on test suites (HumanEval and HumanEval+). No human evaluation of code quality, readability, or correctness is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly separates 'visible test cases' (used during the framework for debugging) from 'hidden test cases, reserved for evaluating the final output' (Section 3)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-model breakdowns for all 19 LLMs across all 6 approaches on both datasets. Figures 4, 5 give detailed per-provider analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.3 discusses cases where ACT hurts performance (e.g., QwQ-Preview where AC/ACT performed significantly worse than Basic). Section 4.3.2 discusses specific failures like agentic complexity reducing accuracy for Llama, DeepSeek, etc."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key negative findings reported: ACT+Debug does not significantly outperform Debug alone (Section 4.2.2); adding a Tester agent often reduces accuracy (Section 4.3.2); ACT+Debug shows highest robustness drop (Section 4.4.1, Figure 11)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about 85% confidence level for combined vs ACT alone, and inability to show significance over Debug alone, are both supported by the t-test results in Section 4.2. Claims about 0.68% improvement match Table 2 analysis."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'combining ACT and debugging improves accuracy' are supported by a controlled ablation design where components are systematically added/removed while other factors are held constant (same models, same datasets, same prompts). This single-variable manipulation is adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Improved Accuracy, Reliability, and Latency' broadly. The conclusion speaks of 'organizations seeking robust AI-driven coding solutions' and 'real-world programming scenarios.' However, results are only on HumanEval/HumanEval+ (Python, function-level tasks). No bounding to this narrow scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper offers one interpretation for each finding (e.g., debugging provides 'rich context') but does not consider confounds such as the additional API calls providing more tokens/attempts regardless of the specific approach, or whether prompt design rather than multi-agent structure drives the differences."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@1 and frames it as 'functional accuracy,' and measures HE/HE+ drop as 'code rigorousness.' These claims match the granularity of the measurements without overclaiming broader code quality constructs."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 lists model names like 'GPT-4o', 'Claude 3.5 Sonnet', 'DeepSeek-V3' without specific version IDs or snapshot dates. The paper only states 'All APIs were accessed in the month of December 2024' which is insufficient per the schema requirement for exact versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Agent prompts are available in the GitHub repository (https://github.com/nazmus-ashrafi/multiagent_vs_debugger), referenced in Sections 3 and 4.3.1: 'All agent prompts, including those used in the debugging process, are can be found in our GitHub repository.'"
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other LLM sampling parameters are reported. Only iteration limits are stated (retriesCT=3, retriesD=4 or 10). These significantly affect output quality."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent scaffolding is described in detail in Section 3 with Figure 1 showing the architecture. Agent roles (Analyst, Coder, Tester), interaction flow, retry logic (retriesCT, retriesD), and the CFG-based debugging mechanism are all documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes how HumanEval is segmented into three components: task description, visible test cases (for execution within framework), and hidden test cases (for final evaluation)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section. Some caveats are mentioned inline (e.g., 'A limitation of this approach is its reliance on a limited set of visible test cases' in Section 3; 'which may not be ideal' regarding same prompts for all models), but these are scattered and not substantive."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "A few specific caveats exist (same prompts for all models, visible test case limitation, n=1 design) but these are brief inline mentions, not substantive discussion. No systematic analysis of what could invalidate the findings."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No mention of limitations to Python, function-level tasks, HumanEval-specific characteristics, or the narrow scope of the tested approaches. The conclusion speaks broadly about 'real-world programming scenarios.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (generated code, execution logs, per-problem pass/fail results) is released. Only aggregated pass@1 scores are shown in Table 2."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are well described: HumanEval (164 tasks, Section 4.1.2) and HumanEval+ (80x more tests). API access timing documented (December 2024, Table 1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (HumanEval, HumanEval+)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The agent pipeline is described but the data pipeline from raw outputs to final metrics is not documented. No information on how many problems each model solved at each stage, how errors were handled, or what intermediate outputs looked like."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers are provided anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: all three authors are from the Department of Computer Science and Software Engineering, United Arab Emirates University. They are not affiliated with any LLM provider being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, making it impossible to assess funder independence. University affiliation suggests academic funding, but without explicit disclosure this cannot be confirmed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states APIs were accessed in December 2024 but does not state training data cutoff dates for any of the 19 models tested."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval/HumanEval+ problems appear in the training data of the 19 models, despite HumanEval being published in 2021 and widely known to be contaminated in newer models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in July 2021. All 19 models tested were trained well after this date and likely saw HumanEval solutions during training. This contamination risk is not addressed anywhere in the paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated benchmark-based."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Latency is reported in detail: Table 3 summarizes average time per approach (7.68 to 68.42 minutes), and Figure 13 shows per-model latency across approaches."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total API costs, GPU hours, or hardware specifications are provided. Running 19 models × 6 approaches × 2 datasets represents substantial compute, but the total budget is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only n=1 (one sample per problem) is generated. No multiple seeds or runs are conducted. Section 4.1.1: 'we chose to generate only one sample per problem (n=1).'"
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated in Section 4.1.1: 'we chose to generate only one sample per problem (n=1) in our experiments.' This is clear, even though it's a single run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The iteration limits (retriesCT=3, retriesD=4/10) appear to be set by design, but no search over LLM hyperparameters (temperature, etc.) is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All six configurations are reported transparently in Table 2 with full per-model results. The paper does not selectively show only the best configuration — all approaches and all models are presented."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple t-tests are conducted (ACT+Debug vs ACT, ACT+Debug vs Debug) without correction for multiple comparisons. The already-lenient α=0.15 exacerbates this issue."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of author-evaluation bias. The authors implement their own version of the ACT and LDB frameworks and compare them, but don't acknowledge that their implementations of baselines may systematically differ from the original authors' implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 (Section 4.5, Table 3) explicitly compares latency vs accuracy across approaches: 'The AC + Debugger configuration attains the highest average accuracy (61.7%) across both datasets while maintaining a reasonable execution time of 38.42 minutes.'"
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval/HumanEval+ actually measures the code generation quality the paper claims to evaluate. These are function-level Python tasks, but the paper claims implications for 'real-world programming scenarios' without questioning the benchmark's validity for that purpose."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The same scaffolding (ACT framework, LDB-based debugger) is applied consistently across all 19 models. Model comparisons use identical prompts and identical agent configurations, controlling for the scaffold variable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021 and all tested models were trained after 2021. Solutions and discussions of HumanEval problems are widely available online. This temporal leakage is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The visible test cases used during the debugging phase provide information about expected behavior. No discussion of whether this constitutes feature leakage relative to real-world usage conditions."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval problems or their variants appeared in model training data. Given the widespread use of HumanEval, near-duplicate problems likely exist in training corpora."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines are used."
       }
     }
   },
   "claims": [
     {
       "claim": "The combined ACT+Debug approach significantly outperforms ACT alone at α=0.15 significance level",
       "evidence": "One-tailed paired t-test across 19 LLMs on HumanEval: mean ACT+Debug 64.82% vs ACT 57.16% (Section 4.2.1, Figure 2). H0,1 rejected.",
       "supported": "moderate"
     },
     {
       "claim": "The combined ACT+Debug approach does not significantly outperform Debug alone",
       "evidence": "One-tailed paired t-test: ACT+Debug 64.82% vs Debug 63.86%, improvement of 0.96%. H0,2 not rejected at α=0.15 (Section 4.2.2, Figure 3).",
       "supported": "strong"
     },
     {
       "claim": "A simple AC (Analyst-Coder) workflow combined with debugging achieves 0.68% mean accuracy improvement over debugging alone",
       "evidence": "Table 2 results across 19 LLMs and two datasets. AC+Debug mean accuracy slightly exceeds Debug mean (Section 4.2.3).",
       "supported": "weak"
     },
     {
       "claim": "Debugging-based approaches generally outperform agentic workflows",
       "evidence": "Mean accuracy: Debug 63.86% vs ACT 57.16% on HumanEval across 19 LLMs. Debug improves over ACT by 6.7% on HumanEval and 7.36% on HumanEval+ (Section 4.3.2).",
       "supported": "moderate"
     },
     {
       "claim": "More complex agentic configurations reduce accuracy when combined with debugging for most models",
       "evidence": "ACT+Debug performs 1.22% worse than AC+Debug on HumanEval+ (Section 4.3.2). Table 2 shows multiple models where ACT+Debug underperforms AC+Debug.",
       "supported": "moderate"
     },
     {
       "claim": "Reduced agentic interaction generally leads to more rigorous code generation",
       "evidence": "Basic approach shows smallest accuracy drop (90.83) between HumanEval and HumanEval+; ACT+Debug shows highest drop (137.74). AC+Debug achieves better HumanEval+ results than ACT+Debug (Section 4.4.1, Figure 11).",
       "supported": "moderate"
     },
     {
       "claim": "OpenAI models generally exhibit stable performance improvements when using combined approaches",
       "evidence": "Figure 4 shows all three OpenAI models (GPT-3.5-turbo, GPT-4o-mini, GPT-4o) improve with combined approaches. But this is only 3 models (Section 4.2.3).",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Very lenient significance threshold",
       "detail": "The study uses α=0.15 instead of the conventional 0.05. The justification (detecting small improvements) is really an argument for larger sample sizes, not lower thresholds. Even at α=0.15, the combined approach fails to significantly outperform debugging alone."
     },
     {
       "flag": "Single-run evaluation (n=1)",
       "detail": "Only one sample is generated per problem with no repeated runs. LLM outputs are stochastic; a single run provides no information about variance. The reported differences (especially 0.68%) could easily fall within run-to-run variation."
     },
     {
       "flag": "No contamination analysis",
       "detail": "HumanEval was published in 2021 and is widely known to be contaminated in post-2021 models. All 19 tested models were trained after this date. The paper draws conclusions about model capabilities without addressing whether models have memorized solutions."
     },
     {
       "flag": "Marginal improvements without adequate statistical support",
       "detail": "The headline finding (0.68% improvement from AC+Debug over Debug) is extremely small and not statistically significant even at α=0.15. Presenting this as a meaningful practical improvement is not well-supported."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations section despite having significant methodological constraints (n=1, two benchmarks only, Python only, no hyperparameter reporting, HumanEval contamination)."
     },
     {
       "flag": "Cherry-picked model narratives",
       "detail": "Section 4.2.3 selectively highlights OpenAI and Anthropic models as 'leading AI research companies' deserving deeper analysis. The narrative around OpenAI models 'generally performing well' is based on only 3 models and appears to be cherry-picked from the mixed overall results."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and pass@k metric, foundational to LLM code generation evaluation."
     },
     {
       "title": "AgentCoder: Multi-Agent-based Code Generation with Iterative Testing and Optimisation",
       "authors": ["D. Huang", "J. M. Zhang", "M. Luck", "Q. Bu", "Y. Qing", "H. Cui"],
       "year": 2023,
       "arxiv_id": "2312.13010",
       "relevance": "Multi-agent framework for code generation with test designer, test executor, and coder agents."
     },
     {
       "title": "MapCoder: Multi-Agent Code Generation for Competitive Problem Solving",
       "authors": ["Md. A. Islam", "M. E. Ali", "M. R. Parvez"],
       "year": 2024,
       "arxiv_id": "2405.11403",
       "relevance": "Multi-agent system with four specialized agents for competitive programming, demonstrating agentic code generation."
     },
     {
       "title": "Self-collaboration Code Generation via ChatGPT",
       "authors": ["Y. Dong", "X. Jiang", "Z. Jin", "G. Li"],
       "year": 2023,
       "arxiv_id": "2304.07590",
       "relevance": "Introduced the Analyst-Coder-Tester framework that this paper directly builds upon and evaluates."
     },
     {
       "title": "Debug like a Human: A Large Language Model Debugger via Verifying Runtime Execution Step-by-step",
       "authors": ["L. Zhong", "Z. Wang", "J. Shang"],
       "year": 2024,
       "arxiv_id": "2402.16906",
       "relevance": "The LDB debugger framework that forms the debugging component of this paper's approach."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["C. Qian"],
       "year": 2023,
       "arxiv_id": "2307.07924",
       "relevance": "Multi-agent software development framework demonstrating role-playing agents for code generation."
     },
     {
       "title": "MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework",
       "authors": ["S. Hong"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent framework for software development using meta-programming approaches."
     },
     {
       "title": "CYCLE: Learning to Self-Refine the Code Generation",
       "authors": ["Y. Ding", "M. J. Min", "G. Kaiser", "B. Ray"],
       "year": 2024,
       "doi": "10.1145/3649825",
       "relevance": "Self-refinement approach for code generation using execution feedback, directly relevant to debugging strategies."
     },
     {
       "title": "RGD: Multi-LLM Based Agent Debugger via Refinement and Generation Guidance",
       "authors": ["H. Jin", "Z. Sun", "H. Chen"],
       "year": 2024,
       "arxiv_id": "2410.01242",
       "relevance": "Multi-LLM debugging framework with Guide, Debug, and Feedback agents for iterative code refinement."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["N. Shinn"],
       "year": 2023,
       "arxiv_id": "2303.11366",
       "relevance": "Verbal reinforcement learning for LLM agents; combined with LDB achieves 98.2 on HumanEval, directly motivating this study."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["J. Liu", "C. Steven Xia", "Y. Wang", "Lingming Zhang"],
       "relevance": "Introduced HumanEval+ with 80x more tests, used as the rigorousness benchmark in this study."
     },
     {
       "title": "SOEN-101: Code Generation by Emulating Software Process Models Using Large Language Model Agents",
       "authors": ["F. Lin", "D. J. Kim", "Tse-Husn", "Chen"],
       "year": 2024,
       "arxiv_id": "2403.15852",
       "relevance": "Multi-agent framework simulating software process models (Waterfall, TDD, Scrum) for code generation."
     },
     {
       "title": "From Code to Correctness: Closing the Last Mile of Code Generation with Hierarchical Debugging",
       "authors": ["Y. Shi", "S. Wang", "C. Wan", "X. Gu"],
       "year": 2024,
       "arxiv_id": "2410.01215",
       "relevance": "Hierarchical debugging tool (MGDebugger) that addresses code bugs at varying granularity levels."
     },
     {
       "title": "Teaching Large Language Models to Self-Debug",
       "authors": ["X. Chen", "M. Lin", "N. Schärli", "D. Zhou"],
       "year": 2023,
       "arxiv_id": "2304.05128",
       "relevance": "Foundational work on LLM self-debugging through iterative generation, explanation, and execution feedback."
     }
   ]
 }