ai-research-survey

scan.json (27473B)

---

 {
   "paper": {
     "title": "Automatically Benchmarking LLM Code Agents through Agent-driven Annotation and Evaluation",
     "authors": [
       "Lingyue Fu",
       "Bolun Zhang",
       "Hao Guan",
       "Yaoming Zhu",
       "Lin Qiu",
       "Weiwen Liu",
       "Xuezhi Cao",
       "Xunliang Cai",
       "Weinan Zhang",
       "Yong Yu"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.24358"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub URL for the PRDBench evaluation code: https://github.com/AGI-Eval-Official/PRDBench (Section 3.1 footnote). A second URL is provided for the minimal code agent: https://github.com/AGI-Eval-Official/Minimal-CodeAgent (Section 4.2 footnote)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'PRDBench data will be available soon' (Section 3.1 footnote), which is a promise of future release. Only the evaluation code is currently available, not the benchmark data itself."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided in the paper. Section 4.3 mentions 'a Python virtual environment that contains necessary and useful packages' but does not list specific package versions or provide an environment file."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While the GitHub repository is referenced, the paper itself does not contain a 'Reproducing Results' section or specific commands to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All main results in Table 3 are reported as single point estimates (e.g., '55.81%', '45.50%') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'commercial agents typically outperform minimal agents') but does not use any statistical significance tests. Comparisons are based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw pass rates and differences (e.g., the 'Enhance' column in Table 3 shows percentage point differences), but no standardized effect sizes like Cohen's d or odds ratios are reported."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark consists of 50 tasks and the human alignment study uses 16 code submissions, but no justification for these sample sizes is provided and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are single-run numbers. No variance, standard deviation, or spread across multiple experimental runs is reported for the main results in Table 3. Variance is mentioned once for alignment scores (Section 4.6.1) but not for the core benchmark results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 8 code agents (4 minimal, 4 commercial) and compares PRDBench against prior benchmarks in Table 1 (SWE-Bench, MLEBench, DevAI, PaperBench)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated agents and models are all current: GPT-5, Claude-3.7-Sonnet, Gemini-2.5-Pro, Qwen3-Coder, Claude Code, CodeX, Gemini CLI, and Qwen Code. These are state-of-the-art systems as of 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7 (Free Development) serves as an ablation-like experiment, comparing performance with and without fixed interfaces/scaffolding. The comparison of minimal vs. commercial agents using the same backbone LLM (Section 4.4, finding 2) also functions as a component analysis."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "PRDBench evaluates across three test types (unit test, shell interaction, file comparison) with separate pass rates reported (Figure 5). The paper also reports development (DEV) and debugging (DEBUG) scores, plus enhancement delta (Table 3), and cost metrics (Table 4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6.1 presents a human alignment study where two annotators scored 282 test cases from 16 code submissions to validate EvalAgent's reliability, achieving 81.56% alignment."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no mention of a held-out test set or dev/test split. All 50 tasks appear to be used for evaluation directly. Since the benchmark is newly constructed, there is no concern about tuning on dev data, but no explicit separation is described."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 provides per-test-type error rates (unit test, shell interaction, file comparison) for each code agent. Figure 4(b) shows domain distribution. Table 5 breaks down human alignment by test type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5.2 discusses error analysis including specific failure patterns. Section 4.4 finding (3) discusses CodeX's 'dramatic drop in performance in the DEBUG phase due to interface inconsistencies introduced during modification.' Section 4.6.1 discusses EvalAgent failure modes including 'propagation chains.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: CodeX experiences performance drops during debugging (Table 3, -6.99 and -3.90 enhancement), and Claude Code scores lower than the minimal ADK-based Claude agent (Section 4.4, finding 2). EvalAgent's instability across projects is also reported honestly."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about reduced annotation cost, diverse evaluation, and Agent-as-a-Judge effectiveness are supported. Table 1 compares benchmarks, Table 3 shows code agent results, and Table 5 validates EvalAgent alignment. The claim of '50 real-world Python projects across 20 domains' is confirmed in Section 4.1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.4 makes causal claims like 'The coding ability of the underlying LLM significantly impacts the development performance of the code agent' based on observational correlation between minimal and commercial agent rankings. The claim 'CodeX experiences a dramatic drop in performance... due to interface inconsistencies' is a causal attribution without controlled analysis isolating this factor."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Automatically Benchmarking LLM Code Agents' broadly, but PRDBench is limited to Python projects only (50 tasks). Section 3.1 acknowledges Python was chosen for 'versatility,' but the generalization to all code agents and languages is not bounded. The abstract claims a 'scalable and robust framework' without caveats."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the correlation between minimal and commercial agent rankings could be explained by factors other than LLM capability (e.g., similar prompting strategies). No threats-to-validity section addresses confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses marketing names without specific version identifiers: 'Claude-3.7-Sonnet', 'Gemini-2.5-Pro', 'GPT-5', 'GPT-4.1', 'GPT-4o', 'Qwen3-Coder-480B-A35B'. No snapshot dates or API version identifiers are provided for any of these models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides full prompts for code agents (Round 1, Round 2, and free development) and for EvalAgent, including the complete prompt text with placeholders and expected output format."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states: 'we set temperature to be 0.1, max token identical to their official APIs' setting. We set top-p to 1.0, Top-k to be 100, and presence penalty to be default to the API.'"
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the agent architecture in detail. Section 3.3 describes EvalAgent's six core tools (file reading/writing, command-line execution, image handling, judge tool). Section 4.2 describes minimal agents using ADK with 'essential tools for file manipulation, bash scripting, and Python execution.' The commercial agents are treated as black boxes, which is appropriate."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the seed task filtering criteria: '(1) the task can be fully implemented in Python; and (2) all datasets required for the task are publicly accessible.' Section 3.2 describes the full 5-step data production workflow with human inspection and iterative refinement. Section 4.1 describes how 50 tasks were selected from filtered candidates across 20 domains."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations or Threats to Validity section in the paper. The conclusion (Section 5) is brief and does not discuss any limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. There is no analysis of how the choice of Python-only tasks, the specific annotator pool, or the use of agent-generated scaffolding might bias results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no discussion of what populations, languages, or settings are excluded from the claims. The conclusion presents PRDBench as a general 'scalable and realistic foundation' without caveats."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark data is not yet available ('PRDBench data will be available soon'). Only the evaluation code is released. Raw experimental outputs (agent logs, generated code) are not available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes seed task sources ('real-world project requirements, including user requests from end-to-end AI product development platforms, academic theses and projects') and Section 3.2 provides the full 5-step agent-driven data production workflow."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A describes the 8 annotators (5 full-time, 3 part-time), listing their educational backgrounds, majors, degree levels, and years of developer experience in Table 7. Compensation method is also mentioned: 'All annotators receive daily wages (part-time) or monthly salaries (full-time) according to local labor regulations.'"
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The 5-step pipeline (Section 3.2) is documented with clear stages: seed task initialization, scaffold and criteria generation, human inspection, agent-based fix and refinement, and scaffold removal. Section 4.1 provides final statistics (50 tasks, 1,262 scoring points broken down by type)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. There is no mention of grants, corporate sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Shanghai Jiao Tong University, Meituan, and AGI-Eval. Meituan is a technology company, and this affiliation is disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. Authors from Meituan (a tech company) could have a commercial interest in benchmark construction tools, but no conflict of interest statement addresses this."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates multiple LLMs (GPT-5, Claude-3.7-Sonnet, Gemini-2.5-Pro, Qwen3-Coder) on the PRDBench benchmark but does not state the training data cutoff dates for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Although PRDBench is newly constructed (reducing contamination risk), the paper does not discuss whether any of the seed tasks, PRDs, or evaluation criteria could have appeared in model training data. No contamination analysis is provided."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "PRDBench is a new benchmark, which reduces contamination risk, but the paper does not explicitly discuss this advantage or verify that benchmark materials were not available before model training cutoffs. The seed tasks come from 'real-world project requirements' and 'academic theses' which may have been publicly available."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The human alignment study in Section 4.6.1 involves annotators evaluating EvalAgent outputs, not a human subjects study in the traditional sense. The annotators are performing quality assurance work, not serving as experimental participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The annotators are hired workers performing annotation tasks, not experimental participants. IRB approval is not typically required for this type of work."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "The annotators are workers performing annotation tasks, not study participants. Their backgrounds are reported in Appendix A for transparency but this is not a human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a research study sense. The annotators are hired workers, not experimental subjects."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study requiring randomization. The paper evaluates code agents, not human behavior."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study requiring blinding. The annotators are validating agent outputs, not serving as experimental subjects."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study with potential attrition. The annotators are hired workers completing assigned tasks."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports detailed cost statistics including time (seconds), input/output token counts, and lines of code for each agent in both rounds. Table 6 reports EvalAgent costs: average 425.62 seconds, 1,242,440 input tokens, 8,825 output tokens, and $2.68 API cost per problem."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 provides comprehensive compute statistics for all agents including wall-clock time and token consumption for both development and debugging phases. Table 6 quantifies EvalAgent's compute costs. Section 4.6.2 also notes '0.5 to 1 hour for human annotators' as a comparison point."
       }
     }
   },
   "claims": [
     {
       "claim": "PRDBench reduces annotation complexity so that annotators with undergraduate-level knowledge can complete annotation, averaging only 8 hours per project.",
       "evidence": "Section 1 states annotators need only 'undergraduate-level knowledge in software engineering related fields' and 'an average of only eight hours needed to finish the scaffolding and metrics for each project.' Appendix A shows annotator backgrounds including Bachelor's degree holders.",
       "supported": "moderate"
     },
     {
       "claim": "The coding ability of the underlying LLM significantly impacts code agent development performance, as minimal and commercial agent rankings are consistent.",
       "evidence": "Table 3 shows that the relative ranking of minimal agents (GPT-5 > Claude > Qwen3-Coder > Gemini) is broadly consistent with commercial agents, though with notable exceptions (Claude Code scores lower than minimal Claude agent).",
       "supported": "moderate"
     },
     {
       "claim": "Debugging and initial development require distinct capabilities from code agents.",
       "evidence": "Table 3 shows CodeX drops from 56.23% (DEV) to 50.24% (DEBUG), while most minimal agents improve. Section 4.4 finding (3) attributes this to 'interface inconsistencies introduced during modification.'",
       "supported": "moderate"
     },
     {
       "claim": "EvalAgent achieves 81.56% alignment with human annotators across 282 test cases.",
       "evidence": "Table 5 reports alignment rates: 81.56% total, 79.44% unit test, 82.55% shell interaction, 84.62% file comparison, based on 16 randomly selected code submissions scored by two annotators.",
       "supported": "moderate"
     },
     {
       "claim": "PRDBench effectively differentiates code agents in both fixed inference and free development modes.",
       "evidence": "Figure 6 shows performance variance of 0.028 for fixed inference vs 0.011 for free development, with 'relative ranking of code agents remains fairly stable between the two settings' (Section 4.7).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-5 is the top-performing minimal agent with 55.81% pass rate in development and 60.15% in debugging.",
       "evidence": "Table 3 shows GPT-5 minimal agent achieves the highest scores in both DEV (55.81%) and DEBUG (60.15%) phases among all agents.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "PRDBench is a new benchmark of 50 Python projects across 20 domains that uses agent-driven annotation to reduce benchmark construction costs, requiring only undergraduate-level annotators averaging 8 hours per project. GPT-5 achieves the highest pass rate (55.81% DEV, 60.15% DEBUG) among 8 evaluated code agents, while debugging and development require distinct agent capabilities, with some agents (CodeX) experiencing performance drops during debugging. The Agent-as-a-Judge (EvalAgent) achieves 81.56% alignment with human scoring across 282 test cases at $2.68 per evaluation, though with substantial variance across projects.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section. Significant limitations such as Python-only scope, small benchmark size (50 tasks), and annotator selection are not discussed."
     },
     {
       "flag": "No statistical rigor in comparisons",
       "detail": "All benchmark results are single-run point estimates with no confidence intervals, error bars, significance tests, or variance across runs. Claims of one agent outperforming another are based solely on comparing two numbers."
     },
     {
       "flag": "Data not yet released",
       "detail": "The benchmark data is described as 'will be available soon' — the paper's core contribution (the benchmark itself) is not independently verifiable at publication time."
     },
     {
       "flag": "Potential conflict of interest",
       "detail": "Authors from Meituan (a major tech company) and AGI-Eval, with no funding disclosure or competing interests statement. The paper evaluates commercial products without disclosing potential conflicts."
     },
     {
       "flag": "Small human alignment sample",
       "detail": "The EvalAgent validation study uses only 16 code submissions (282 test cases) with only 2 annotators. The high variance reported (std dev 27.83%, range 0-100%) undermines the 81.56% alignment headline, yet no discussion of this limitation appears."
     },
     {
       "flag": "Unbounded generalization claims",
       "detail": "The paper presents PRDBench as a 'scalable and robust framework' for code agent evaluation generally, but tests only Python projects. The title claims to benchmark 'LLM Code Agents' without language qualification."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Foundational benchmark for evaluating code agents on real GitHub issues, directly compared against PRDBench."
     },
     {
       "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering",
       "authors": ["Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe"],
       "year": 2024,
       "arxiv_id": "2410.07095",
       "relevance": "Benchmark for evaluating ML engineering agents, compared against PRDBench in Table 1."
     },
     {
       "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research",
       "authors": ["Giulio Starace", "Oliver Jaffe", "Dane Sherburn"],
       "year": 2025,
       "arxiv_id": "2504.01848",
       "relevance": "High-cost benchmark using PhD-level annotators, directly motivated PRDBench's lower-cost annotation approach."
     },
     {
       "title": "Agent-as-a-Judge: Evaluate Agents with Agents",
       "authors": ["Mingchen Zhuge", "Changsheng Zhao", "Dylan Ashley"],
       "year": 2024,
       "arxiv_id": "2410.10934",
       "relevance": "Introduces the Agent-as-a-Judge paradigm that PRDBench adopts for evaluation, the DevAI benchmark."
     },
     {
       "title": "CoreCodeBench: A Configurable Multi-Scenario Repository-Level Benchmark",
       "authors": ["Lingyue Fu", "Hao Guan", "Bolun Zhang"],
       "year": 2025,
       "arxiv_id": "2507.05281",
       "relevance": "Repository-level code benchmark from same research group, related benchmark construction approach."
     },
     {
       "title": "Automated Benchmark Generation for Repository-Level Coding Tasks",
       "authors": ["Konstantinos Vergopoulos", "Mark Niklas Müller", "Martin Vechev"],
       "year": 2025,
       "arxiv_id": "2503.07701",
       "relevance": "Automated benchmark generation approach for repository-level coding, directly related methodology."
     },
     {
       "title": "CodeVisionary: An Agent-based Framework for Evaluating Large Language Models in Code Generation",
       "authors": ["Xinchen Wang", "Pengfei Gao", "Chao Peng"],
       "year": 2025,
       "arxiv_id": "2504.13472",
       "relevance": "Agent-based LLM code evaluation framework, explores LLM-as-judge for code generation assessment."
     },
     {
       "title": "Deep-Bench: Deep Learning Benchmark Dataset for Code Generation",
       "authors": ["Alireza Daghighfarsoodeh", "Chung-Yu Wang"],
       "year": 2025,
       "arxiv_id": "2502.18726",
       "relevance": "Benchmark for code generation in deep learning tasks, complementary evaluation of LLM coding capabilities."
     },
     {
       "title": "ProjectEval: A Benchmark for Programming Agents Automated Evaluation on Project-Level Code Generation",
       "authors": ["Kaiyuan Liu", "Youcheng Pan"],
       "year": 2025,
       "arxiv_id": "2503.07010",
       "relevance": "Project-level code generation benchmark that simulates user interaction, alternative evaluation approach for code agents."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2025,
       "arxiv_id": "2406.15877",
       "relevance": "Benchmark for code generation with diverse function calls, assessing LLM coding capabilities."
     },
     {
       "title": "SciReplicate-Bench: Benchmarking LLMs in Agent-driven Algorithmic Reproduction from Research Papers",
       "authors": ["Yanzheng Xiang", "Hanqi Yan"],
       "year": 2025,
       "arxiv_id": "2504.00255",
       "relevance": "Benchmark for agent-driven research paper reproduction, related evaluation of code agent capabilities."
     },
     {
       "title": "LLM-based Agents Suffer from Hallucinations: A Survey of Taxonomy, Methods, and Directions",
       "authors": ["Xixun Lin", "Yucheng Ning"],
       "year": 2025,
       "arxiv_id": "2509.18970",
       "relevance": "Survey on LLM agent hallucinations, cited for error propagation chains observed in EvalAgent."
     }
   ]
 }