ai-research-survey

scan.json (29306B)

---

 {
   "paper": {
     "title": "An Agent-based Evaluation Framework for Complex Code Generation",
     "authors": [
       "Xinchen Wang",
       "Ruida Hu",
       "Pengfei Gao",
       "Chao Peng",
       "Cuiyun Gao"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2504.13472"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states: 'The resources of CodeVisionary are available at https://github.com/Eshe0922/CodeVisionary.' A working GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark of 363 samples was constructed by the authors from CodeArena filtered to 'hard' tasks with human annotations. No link to download this constructed benchmark is provided in the paper; only the GitHub link for the framework code is mentioned."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using Docker containers for code execution within CodeVisionary, but does not provide a requirements file, Dockerfile for the framework itself, or a detailed environment setup section listing library versions needed to run the evaluation framework."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper references the GitHub repository but provides no step-by-step instructions for reproducing the main experimental results in the paper text itself. It is unclear from the paper whether the repository contains such instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All main results in Table II and Table III are reported as single point estimates (Pearson, Spearman, Kendall-Tau coefficients) with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CodeVisionary outperforms baselines based on comparing numeric coefficients (e.g., 0.301 vs. 0.084 average rp) but performs no statistical significance tests to validate these differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both raw correlation coefficients and absolute improvements (e.g., 'outperforming the best baseline with average improvements of 0.217, 0.163, and 0.141 in Pearson, Spearman, and Kendall-Tau coefficients'), providing sufficient context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses 363 samples derived from filtering CodeArena's 397 'hard' tasks, but there is no power analysis or justification for why this sample size is sufficient for the correlation-based claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we perform multiple trials and take the average as the experimental results' (Section V-D) but does not report standard deviation, IQR, or any spread measure across these trials. Only average values appear in the result tables."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared: VANILLA (direct LLM prompting), ICE-Score, and CODEJUDGE, representing naive, and two state-of-the-art LLM-based code evaluation approaches."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "ICE-Score (EACL 2024) and CODEJUDGE (EMNLP 2024) are both recent contemporary baselines. The authors state they 'try our best to reproduce them from publicly available source code and papers.'"
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B (RQ2) presents ablation studies: removing the RMCD stage (w/o RMCD), removing the FSAS stage (w/o FSAS), and removing individual information types (w/o RT, w/o UI/UX, w/o ST) from the RMCD stage, with results in Table III."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three correlation metrics are used: Pearson (rp), Spearman (rs), and Kendall-Tau (tau) coefficients, all measuring correlation with human-annotated ground truth scores."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The human expert annotations create ground truth labels for the benchmark. They evaluate the LLM-generated code quality, NOT CodeVisionary's outputs. Per the schema: 'The humans must be evaluating what the system produced -- manual classification of the benchmark or dataset itself does not count.' No humans evaluated CodeVisionary's evaluation outputs (scores, reports). Evaluation of the system is entirely automated via correlation metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "RQ3 explores hyperparameter choices (number of judges 2-5, rounds 2-5) on the same 363-sample test set that final results are reported on. The schema states 'If unclear whether the reported numbers are on data used for any selection decisions, NO.' The same dataset is used for both hyperparameter selection and evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 provides per-coding-scenario breakdowns (Development and Programming, Tools Environments and Practices, UI/UX, Emerging Technologies, Algorithm Design) and per-programming-language breakdowns (Python, JavaScript, Java, C#, C)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses failure cases of the BASELINES (ICE-Score) in Section I to motivate the work. However, it does not discuss where CodeVisionary itself fails or breaks down. The case study (Figure 5) shows a success case. The schema requires 'discussion of where the approach breaks down.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C tests CodeVisionary on the less complex CoNaLa benchmark and reports that it does not outperform ICE-Score on rp (0.644 vs 0.655), which is an honest negative result. The ablation in Table III also shows performance degradation when components are removed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'average improvements of 0.217, 0.163, and 0.141 in Pearson, Spearman, and Kendall-Tau coefficients' and Table II confirms CodeVisionary achieves avg 0.301/0.272/0.241 vs. best baseline 0.084/0.109/0.100. The claim is supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'The RMCD stage boosts rp, rs, and tau by 27.0%, 32.0%, and 32.4%' based on ablation studies, but the ablation design removes entire stages rather than isolating individual variables precisely. Additionally, the same dataset is used for hyperparameter selection (RQ3) and final evaluation, creating potential circularity."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames contributions broadly ('the first agent-based evaluation framework for complex code generation') but tests only on one benchmark (CodeArena hard tasks, 363 samples) and one simpler dataset (CoNaLa). The paper acknowledges in the threats section that 'our benchmark may not cover all coding scenarios and programming languages' but the title and abstract do not bound the claims accordingly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats section (Section V-D) mentions only two threats: benchmark coverage limitation and LLM randomness. The paper does not discuss alternative explanations for the improvements, such as whether the advantage stems primarily from GPT-4o having access to more computation (via 40 interactions) rather than the architectural design per se."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'CodeVisionary and the baseline methods are provided access to GPT-4o' (Section III-D), but uses only the marketing name 'GPT-4o' without a specific API version or snapshot date. The references for GPT-3.5-turbo, Claude-3.5-Sonnet, and GPT-4o point to product announcement pages rather than specific model versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompts in natural language (e.g., the 'thought'/'action' paradigm, evaluation criteria) and provides one example of clarity criteria, but the full prompts sent to GPT-4o are not included in the paper text. The paper refers readers to the repository for 'remaining aspects and criteria' but actual prompt text is not reproduced."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section III-D reports: temperature=0.2 for RMCD stage, temperature=0.7 for FSAS stage, max interactions=40 for RMCD, number of judges=3, max negotiation rounds=4. These key hyperparameters are specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively describes the agentic scaffolding: Docker container setup, tool list (Dynamic Execution, Static Linter, Unit Tests, Screenshot, Interaction, Web Browsing, General Semantic, Bash Command), the thought/action/observation loop, the Execute/Analyze state alternation, and the multi-judge negotiation protocol with formal definitions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A documents the benchmark construction pipeline: (1) filter CodeArena to 'hard' tasks, (2) exclude platform-specific tasks (MATLAB, Verilog), (3) generate responses using 3 LLMs, (4) manual scoring by two experts with Kappa>80% and third-expert adjudication. The filtering criteria are stated explicitly."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V-D is titled 'Threats and Limitations' and provides dedicated discussion of threats to validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section V-D mentions only two threats: (1) benchmark coverage may not generalize and (2) LLM randomness. Both are generic disclaimers. The threats do not discuss specific concerns like confounding from compute asymmetry between CodeVisionary and baselines, or the fact that the same data is used for both hyperparameter selection and evaluation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section notes the benchmark 'may not cover all coding scenarios and programming languages' but does not explicitly state what the results do NOT show (e.g., does not state results are limited to GPT-4o as evaluator, or to 'hard' complexity tasks, or to the specific benchmark construction methodology)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not provide a download link for the 363-sample benchmark with human annotations. Only the framework code repository is referenced. The human-annotated benchmark data needed to reproduce the correlation results is not publicly available based on the paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes the data collection: filtering CodeArena to hard tasks, excluding platform-specific tasks, generating responses via GPT-3.5-turbo/Claude-3.5-Sonnet/GPT-4o, and human annotation procedure with inter-rater reliability (Kappa>80%)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states expert annotators have 'over five years of expertise in the relevant programming languages' but does not describe how these experts were recruited, selected, or compensated, or whether they could introduce selection bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline from CodeArena (397 tasks) to final benchmark (363 samples with 121 tasks x 3 LLM responses) is documented: filtering hard tasks, excluding certain platform-specific ones, generating responses, manual scoring. The counts are consistent (121 tasks x 3 = 363 samples)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section discloses funding: National Natural Science Foundation of China (No. 62472126, 62276075), Natural Science Foundation of Guangdong Province (No. 2023A1515011959), and Shenzhen-Hong Kong Jointly Funded Project (No. SGDX20230116091246007)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the first page: Xinchen Wang and Ruida Hu are from Harbin Institute of Technology, Shenzhen; Pengfei Gao and Chao Peng are from ByteDance. A footnote notes 'Work done during an internship at ByteDance.' The industry affiliation is disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are the National Natural Science Foundation of China and a Shenzhen-Hong Kong government project — public research funding with no financial interest in CodeVisionary's performance. ByteDance is an industrial affiliate of some authors but is not listed as a funder."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement or declaration that authors hold (or do not hold) patents or equity related to the work. The absence of such a statement means this criterion is not met."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o as the evaluator model but does not state GPT-4o's training data cutoff date. It also evaluates responses generated by GPT-3.5-turbo, Claude-3.5-Sonnet, and GPT-4o without stating any model's training cutoff."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is derived from CodeArena which was published in 2024 and could have been in the training data of the models used. The paper does not discuss whether the code generation tasks could have appeared in model training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CodeArena was published in late 2024 (arXiv:2412.05210) and GPT-4o's training cutoff is unknown from the paper. The paper does not discuss contamination risk for either the benchmark tasks or the generated responses being evaluated."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The expert annotators are providing ground truth labels for a benchmark -- they are professional raters, not research participants in a human subjects study. Pre-registration applies to studies OF human participants, not studies that use human annotators as instruments. This is a benchmark evaluation paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "Expert annotators creating benchmark labels are not human research participants. IRB review is for research ON human subjects, not for employing human annotators in a data pipeline. This is a benchmark evaluation paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "The annotators are expert raters creating ground truth labels, not participants in a human study. Demographic reporting requirements apply to studies of human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning -- expert annotators are instruments for benchmark construction, not human research participants. Inclusion/exclusion criteria requirements are for participant recruitment in human subjects research."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The human annotation study is an observational scoring study, not an experimental study with treatment/control assignment. Randomization of annotators to conditions does not apply."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. While the paper does note that experts 'remain unaware of the identity of the LLM,' this is good annotation practice, not blinding in a human subjects study context."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "There is no longitudinal study with dropout risk. The annotation was a bounded task and attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "CodeVisionary uses GPT-4o with up to 40 interactions per evaluation instance in the RMCD stage plus additional calls in the FSAS stage. Table IV reports an average of 6.12 actions per instance, but no API cost, token count, or monetary cost is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (GPU hours, API spend, wall-clock time) is stated for running the 363-sample evaluation. Given the 40-interaction limit per instance with GPT-4o, this is a non-trivial cost that is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeVisionary achieves the best performance among three baselines for evaluating complex code generation, outperforming the best baseline with average improvements of 0.217, 0.163, and 0.141 in Pearson, Spearman, and Kendall-Tau coefficients, respectively.",
       "evidence": "Table II shows CodeVisionary avg rp=0.301, rs=0.272, tau=0.241 versus best baseline (CODEJUDGE avg rp=0.084, rs=0.094, tau=0.089). Section IV-A discusses results.",
       "supported": "strong"
     },
     {
       "claim": "The requirement-guided multi-dimensional context distillation (RMCD) stage improves CodeVisionary performance by an average of 27.0%, 32.0%, and 32.4% across rp, rs, and tau respectively.",
       "evidence": "Table III ablation results: w/o RMCD achieves avg rp=0.237, rs=0.206, tau=0.182 vs. full system rp=0.301, rs=0.272, tau=0.241. Section IV-B1.",
       "supported": "moderate"
     },
     {
       "claim": "The fine-grained scoring and summarization (FSAS) stage improves CodeVisionary performance by 21.4%, 21.4%, and 19.9% on average across rp, rs, and tau.",
       "evidence": "Table III: w/o FSAS achieves avg rp=0.248, rs=0.224, tau=0.201 vs. full system. Section IV-B2.",
       "supported": "moderate"
     },
     {
       "claim": "Runtime information (dynamic execution and unit tests) provides the most contribution, improving rp, rs, and tau by 46.8%, 40.9%, and 41.8% respectively.",
       "evidence": "Table III: w/o RT achieves avg rp=0.205, rs=0.193, tau=0.170 vs. full system. Section IV-B3.",
       "supported": "moderate"
     },
     {
       "claim": "Current state-of-the-art LLM-based evaluation approaches (ICE-Score, CODEJUDGE) exhibit minimal difference compared to a vanilla baseline for complex code evaluation.",
       "evidence": "Table II: VANILLA avg rp/rs/tau = 0.042/0.091/0.085; ICE-SCORE = 0.039/0.109/0.100; CODEJUDGE = 0.084/0.094/0.089. Section IV-A.",
       "supported": "strong"
     },
     {
       "claim": "CodeVisionary maintains effectiveness on less complex tasks, achieving competitive performance on CoNaLa.",
       "evidence": "Table V: CodeVisionary rp=0.644, rs=0.637, tau=0.572 vs. ICE-SCORE rp=0.655, rs=0.596, tau=0.534. CodeVisionary is better on rs and tau but slightly worse on rp. Section V-C.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "CodeVisionary, a two-stage agent-based framework, substantially outperforms existing LLM-based code evaluation methods (ICE-Score, CODEJUDGE) on complex code generation tasks, with average Pearson correlation improvements of 0.217 over the best baseline. The framework's advantage stems from combining multi-dimensional context collection (runtime execution, static linting, web browsing, UI/UX screenshots) with a multi-judge negotiation protocol. Notably, existing state-of-the-art LLM-based evaluation methods show only marginal improvement over a vanilla prompting baseline on complex code tasks, highlighting the difficulty of this evaluation setting. Performance on a simpler benchmark (CoNaLa) is competitive but does not uniformly exceed ICE-Score, suggesting the approach is most valuable for complex scenarios.",
   "red_flags": [
     {
       "flag": "No model version specificity",
       "detail": "The paper uses 'GPT-4o' throughout without specifying API version or snapshot date. GPT-4o behavior changes across API versions, making exact reproduction impossible. References for the model point only to announcement blog posts."
     },
     {
       "flag": "Compute asymmetry between method and baselines",
       "detail": "CodeVisionary uses up to 40 interactions per evaluation instance with GPT-4o, plus additional FSAS calls, while baselines use a single or few LLM calls. The improvements may largely reflect more compute spent per evaluation rather than the architectural novelty. No cost or token count is reported to assess this."
     },
     {
       "flag": "Same dataset for hyperparameter selection and evaluation",
       "detail": "RQ3 explores hyperparameter choices (number of judges 2-5, number of rounds 2-5) on the same 363-sample test set, then reports final results on that same set. This conflates hyperparameter optimization and evaluation, potentially inflating reported performance."
     },
     {
       "flag": "No variance reported across multiple trials",
       "detail": "The paper states multiple trials are averaged but reports no standard deviation or confidence intervals. The magnitude of the improvements is large, but without variance estimates, the reliability of the differences cannot be assessed."
     },
     {
       "flag": "Benchmark annotation data not released",
       "detail": "The human-annotated benchmark of 363 samples (the ground truth for all correlation computations) is not publicly available, making it impossible to independently verify the reported correlation results."
     },
     {
       "flag": "Industry co-authorship without COI declaration",
       "detail": "Two authors (Pengfei Gao, Chao Peng) are from ByteDance, and two student authors did internships at ByteDance. CodeVisionary is presented as a tool that could be commercially deployed. No competing interests statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "ICE-Score: Instructing Large Language Models to Evaluate Code",
       "authors": [
         "T. Y. Zhuo"
       ],
       "year": 2024,
       "relevance": "Direct baseline for the paper's code evaluation framework; a representative LLM-based code evaluation approach."
     },
     {
       "title": "CodeJudge: Evaluating Code Generation with Large Language Models",
       "authors": [
         "W. Tong",
         "T. Zhang"
       ],
       "year": 2024,
       "relevance": "Direct baseline for the paper; uses 'slow thinking' approach for semantic correctness evaluation of generated code."
     },
     {
       "title": "Evaluating and Aligning CodeLLMs on Human Preference",
       "authors": [
         "J. Yang",
         "J. Yang",
         "K. Jin",
         "Y. Miao",
         "L. Zhang",
         "L. Yang",
         "Z. Cui",
         "Y. Zhang",
         "B. Hui",
         "J. Lin"
       ],
       "year": 2024,
       "arxiv_id": "2412.05210",
       "relevance": "Source of the CodeArena benchmark used to construct CodeVisionary's evaluation dataset."
     },
     {
       "title": "Out of the BLEU: how should we assess quality of the code generation models?",
       "authors": [
         "M. Evtikhiev",
         "E. Bogomolov",
         "Y. Sokolov",
         "T. Bryksin"
       ],
       "year": 2023,
       "relevance": "Human evaluation study of code generation quality; provides the human annotation framework used for scoring in this paper."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": [
         "M. Chen",
         "J. Tworek",
         "H. Jun"
       ],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval and pass@k metric, foundational benchmark evaluation methodology for code generation."
     },
     {
       "title": "A survey on evaluating large language models in code generation tasks",
       "authors": [
         "L. Chen",
         "Q. Guo",
         "H. Jia"
       ],
       "year": 2024,
       "arxiv_id": "2408.16498",
       "relevance": "Survey of code generation evaluation approaches, providing context for the paper's contributions."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": [
         "L. Zheng",
         "W.-L. Chiang",
         "Y. Sheng"
       ],
       "year": 2023,
       "relevance": "Foundational paper on using LLMs as judges for evaluation, relevant to the LLM-as-evaluator paradigm."
     },
     {
       "title": "The rise and potential of large language model based agents: A survey",
       "authors": [
         "Z. Xi",
         "W. Chen",
         "X. Guo"
       ],
       "year": 2025,
       "relevance": "Comprehensive survey on LLM-based agents, providing theoretical grounding for the agent-based approach used."
     },
     {
       "title": "Large language model-based agents for software engineering: A survey",
       "authors": [
         "J. Liu",
         "K. Wang",
         "Y. Chen"
       ],
       "year": 2024,
       "arxiv_id": "2409.02977",
       "relevance": "Survey of LLM agents applied to software engineering tasks, relevant to the paper's positioning."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": [
         "C. Qian",
         "W. Liu",
         "H. Liu"
       ],
       "year": 2024,
       "relevance": "Multi-agent framework for software development; related to the multi-agent paradigm used in CodeVisionary."
     },
     {
       "title": "Can LLMs replace human evaluators? An empirical study of LLM-as-a-judge in software engineering",
       "authors": [
         "R. Wang",
         "J. Guo",
         "C. Gao"
       ],
       "year": 2025,
       "relevance": "Empirical study of LLM-as-judge for software engineering evaluation, directly relevant to this paper's core topic."
     },
     {
       "title": "AgentCoder: Multi-agent-based code generation with iterative testing and optimisation",
       "authors": [
         "D. Huang",
         "Q. Bu",
         "J. M. Zhang"
       ],
       "year": 2023,
       "arxiv_id": "2312.13010",
       "relevance": "Multi-agent code generation system; relevant as a precursor to agent-based approaches in software engineering."
     },
     {
       "title": "CodePlan: Repository-level coding using LLMs and planning",
       "authors": [
         "R. Bairi",
         "A. Sonwane",
         "A. Kanade"
       ],
       "year": 2024,
       "relevance": "Repository-level code generation requiring complex multi-step reasoning, motivating evaluation of complex code scenarios."
     }
   ]
 }