ai-research-survey

scan.json (26566B)

---

 {
   "paper": {
     "title": "ProjDevBench: Benchmarking AI Coding Agents on End-to-End Project Development",
     "authors": ["Pengrui Lu", "Shiqi Zhang", "Yunzhong Hou", "Lyumanshan Ye", "Chaoyi Huang", "Zixi Chen", "Ji Zeng", "Hantao Jiang", "Pengfei Liu", "Yiwei Wang", "Ming-Hsuan Yang"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.01655"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ProjDevBench evaluates 6 coding agents across multiple LLM backends on 20 end-to-end C++ project construction tasks, finding an overall acceptance rate of only 27.38%. Codex+GPT-5 achieves the best overall score (77.85%), with performance gaps widening on from-scratch tasks. Key failure modes include specification misalignment (42% wrong answers), time complexity optimization failures (14% TLE), and resource management limitations. Extended interaction correlates negatively with performance (Spearman ρ = −0.734 for tokens vs. score).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub URL: https://github.com/zsworld6/projdevbench in the abstract."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark problems are available via the GitHub repository mentioned in the abstract."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E provides a detailed Docker base image configuration (Dockerfile) specifying exact packages: gcc-13, g++-13, cmake, Python 3.12, Node.js 20, and all agent CLI tools."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.2 documents the full evaluation pipeline including configuration loading, Docker container initialization, workspace setup, agent execution, OJ submission, and code review steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2–6 report only point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H mentions a paired t-test: 'Pairwise comparison of top agents (Cursor vs. Claude Code) shows statistically significant difference (p < 0.05, paired t-test).' However, this is only for one comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage scores with baselines for comparison (e.g., Codex+GPT-5 at 77.85 vs. Augment+GPT-5 at 72.35), and Spearman correlations with specific ρ values and p-values in Table 4."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has only 20 problems with no justification for why 20 is sufficient. No power analysis or discussion of statistical adequacy of this sample size for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Single evaluation pass per agent-model configuration. No variance across runs reported. Appendix H reports std of 32.4 across problems, but this is cross-problem variation, not cross-run variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six coding agents across multiple LLM backends are compared against each other, and Table 1 compares ProjDevBench against prior benchmarks (HumanEval, SWE-bench, DevEval, etc.)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated agents and models are highly contemporary: GPT-5, Claude Sonnet 4.5, Gemini 3 Pro Preview, plus recent open-source models. All are 2025-2026 era."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study of the benchmark design (e.g., effect of code review weight, different scoring schemes). The 80/20 execution/review weighting is asserted without ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses three metrics: Execution Score, Code Review Score, and Final Weighted Score, reported separately in Table 2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 validates the LLM-based code review against human expert annotations, with multiple annotators independently reviewing submissions. Figure 5 shows Pearson r=0.709 and Cohen's κ=0.710."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss held-out test sets. All 20 problems are used for evaluation, and the OJ test suites serve as the evaluation mechanism. No train/dev/test split is mentioned."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 breaks results down by Easy vs. Hard subsets. Table 3 provides per-failure-type distribution. Table 6 reports per-problem scores. Figure 4 shows category distribution."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 provides extensive failure analysis across five categories: specification misalignment, edge case handling, time complexity, resource management, and code engineering gaps, with specific examples from individual problems."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively reports where agents fail: only 27.38% acceptance rate, agents struggle with complex tasks, extended interaction correlates negatively with performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 27.38% acceptance rate (supported by Table 3), agents struggle with complex system design (supported by Section 5.1), and the benchmark evaluates system architecture, correctness, and refinement (supported by the evaluation protocol)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims such as 'extended interaction alone does not guarantee successful task completion' and attributes performance differences to framework design vs. model capability without controlling for confounds. The correlation analysis (Table 4) is acknowledged as correlational, but language like 'harder problems compel agents' implies causation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states: 'tasks focus primarily on C++, and it remains unclear whether observed agent behaviors generalize to other languages.' The scope is bounded to the 20 tasks and tested agents."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for performance differences. For example, agent framework differences (prompting strategies, tool usage) are not controlled for, and no discussion of whether the OJ platform characteristics bias results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is relatively precise: it measures execution scores and code review compliance, and frames these as measures of 'end-to-end project construction capability' — which is what the benchmark directly tests. The metrics match the claims without overframing."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are specified by marketing names only: 'GPT-5', 'Sonnet-4.5', 'Gemini 3 Pro Preview', 'GLM-4.6', 'Kimi-k2-0905-Preview', 'DeepSeek-V3.2-Exp'. No API versions or snapshot dates are provided except Kimi which includes a date-like suffix."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F provides the full prompt text used for both single-problem and multiple-problem evaluation scenarios."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of temperature, top-p, max tokens, or any sampling hyperparameters for the LLM API calls used by the agents."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates third-party coding agents (Cursor, Copilot, Claude Code, etc.) as black boxes via their CLIs. The authors cannot describe the internal scaffolding of these proprietary tools."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 and Figure 3 document the three-stage problem filtering pipeline: ~2800 initial → ~100 after scope filtering → 20 after quality filtering, with criteria described at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations and Future Work' section appears after Section 6, discussing benchmark scale, language coverage, and evaluation mode limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section raises specific concerns: only 20 tasks (scaling difficulty), C++ only (language generalization unclear), fully autonomous only (excludes human-in-the-loop). These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what is not covered: other programming languages, human-in-the-loop workflows, and larger task sets. The Limitations section bounds the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the benchmark problems are released, the raw agent execution logs, submission histories, and OJ test outputs are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the three-stage collection pipeline from a university OJ platform, with clear filtering criteria at each stage."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The study evaluates AI agents on curated problems. Student performance statistics in Table 5 are historical data from the OJ platform."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from ~2800 problems → ~100 → 20 is documented in Section 3.2 with filtering criteria. The evaluation pipeline is documented in Appendix E.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: UC Merced, Shanghai Jiao Tong University, Shanghai Innovation Institute, Beijing Institute of Technology. None of these are the evaluated tool vendors."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the evaluated models (GPT-5, Sonnet 4.5, Gemini 3 Pro, etc.)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the OJ problems or their solutions may have appeared in the models' training data. The problems come from a university OJ platform that may have solutions online."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark problems are drawn from an existing OJ platform with ~2800 problems. No discussion of whether these problems and their solutions were available online before the models' training cutoffs."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates AI coding agents on benchmark problems."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports token consumption: agents average 4.81M tokens per problem (Section 5.3), with per-problem token counts in Table 6. Time information is also provided (most complex tasks take up to two hours)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.3 states resource limits: 8GB memory, 4 CPU cores, 6GB Node.js heap per container. Per-problem token budgets are in Table 6."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Single evaluation pass per agent-model configuration. No multiple seeds or runs reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly states: 'For each agent-model configuration, we run a single evaluation pass on every problem in the benchmark.'"
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates coding agents via their CLIs without hyperparameter tuning. No search budget applies."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.3 states the final reported score is the maximum across all valid submissions within the submission limit, and this policy is transparently described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple agent-model comparisons are made across Table 2 but only one p-value is reported (Appendix H) with no multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors developed the benchmark and evaluate all agents on it. No discussion of author-evaluation bias or whether the benchmark design may favor certain agent types."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 and Section 5.3 analyze the relationship between compute (tokens) and performance, finding a strong negative correlation (ρ = −0.734)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the 20 OJ-derived C++ problems actually measure 'end-to-end project development capability' as claimed. No analysis of construct validity or comparison with alternative definitions of project-level development."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates bundled tools (Cursor, Copilot, Claude Code) as products. The scaffold IS what is being tested, so this confound does not apply."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Problems come from an existing university OJ platform. No discussion of whether solutions to these problems existed online before the models' training cutoffs."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. Agents have internet access during evaluation (Appendix E.3: 'Containers have full internet access'), which could allow looking up solutions."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 20 problems share structural similarities that could inflate or deflate performance estimates."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are described despite using problems from a public OJ platform and giving agents internet access."
       }
     }
   },
   "claims": [
     {
       "claim": "Overall acceptance rate across all agents is 27.38%",
       "evidence": "Table 3 shows 484 out of 1,768 total submissions were accepted (27.38%).",
       "supported": "strong"
     },
     {
       "claim": "Codex+GPT-5 achieves the best overall performance at 77.85% final weighted score",
       "evidence": "Table 2 shows Codex+GPT-5 at 77.85 overall final score, ahead of all other configurations.",
       "supported": "moderate"
     },
     {
       "claim": "Extended interaction correlates negatively with performance",
       "evidence": "Table 4 reports Spearman ρ = −0.734 (p = 0.0002) for tokens vs. score and ρ = −0.668 (p = 0.0013) for turns vs. score, across 20 problems in Claude Code.",
       "supported": "moderate"
     },
     {
       "claim": "42% of failures are due to wrong answers and 14% to time limit exceeded",
       "evidence": "Table 3 shows Wrong Answer at 41.86% and Time Limit Exceeded at 13.91% of submissions.",
       "supported": "strong"
     },
     {
       "claim": "LLM-based code review aligns closely with human judgment (accuracy 0.852, Cohen's κ 0.710)",
       "evidence": "Section 5.4 and Figure 5 report human validation with multiple annotators showing Pearson r=0.709 for continuous assessment and accuracy 0.852, κ=0.710 for binary rule verification.",
       "supported": "moderate"
     },
     {
       "claim": "Performance gaps widen on from-scratch construction tasks compared to project-completion tasks",
       "evidence": "Table 2 shows several agents dropping substantially from Easy to Hard (e.g., GitHub Copilot+Sonnet-4.5 from 71.10 to 36.63 execution, Gemini CLI from 74.57 to 35.53).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Single run per configuration",
       "detail": "Each agent-model pair is evaluated once per problem with no repeated runs. Given the stochastic nature of LLM outputs, results could differ substantially across runs, but no variance is reported."
     },
     {
       "flag": "Very small benchmark (N=20)",
       "detail": "Only 20 problems, with results aggregated across them. Statistical conclusions drawn from 20 data points have limited power, and individual problem characteristics could drive aggregate results."
     },
     {
       "flag": "Contamination risk from public OJ platform",
       "detail": "Problems are sourced from a university OJ platform with ~2800 problems. Solutions may be publicly available online, and agents have full internet access during evaluation. No contamination analysis is performed."
     },
     {
       "flag": "Internet access during evaluation",
       "detail": "Appendix E.3 states 'Containers have full internet access for network operations.' Agents could potentially look up solutions to OJ problems during evaluation, confounding the measurement of coding capability."
     },
     {
       "flag": "C++ only",
       "detail": "All 20 problems are C++ tasks, yet the paper draws conclusions about 'end-to-end project development' capability broadly. Agent performance in C++ may not reflect capability in more commonly used languages like Python or JavaScript."
     },
     {
       "flag": "Uncontrolled agent framework confound",
       "detail": "Different agents use different scaffolding, tool access, and interaction strategies. Comparing Codex+GPT-5 vs. Claude Code+GPT-5 conflates framework effects with other differences. The interaction analysis (Table 4, Table 6) is done only on Claude Code, limiting generalizability."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Introduced HumanEval, the foundational function-level code generation benchmark that ProjDevBench extends beyond."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Key issue-resolution benchmark that ProjDevBench positions against as insufficient for end-to-end development evaluation."
     },
     {
       "title": "OpenHands: An open platform for AI software developers as generalist agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2025,
       "arxiv_id": "2407.16741",
       "relevance": "Open-source autonomous coding agent framework representative of the agents ProjDevBench evaluates."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Explores agent-computer interfaces for SE, directly relevant to understanding how agents interact with development environments."
     },
     {
       "title": "Agentless: Demystifying LLM-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Contrasts agentic vs. non-agentic approaches to SE, relevant baseline approach."
     },
     {
       "title": "Self-refine: iterative refinement with self-feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "relevance": "Iterative self-refinement paradigm that ProjDevBench's multi-submission evaluation tests."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "relevance": "Competition-level code generation benchmark, predecessor to project-level evaluation."
     },
     {
       "title": "DevEval: A manually-annotated code generation benchmark aligned with real-world code repositories",
       "authors": ["Juyong Li", "Guanjun Li", "Yongmin Zhao"],
       "year": 2024,
       "arxiv_id": "2405.19856",
       "relevance": "Repository-aligned benchmark with staged development, directly compared against in Table 1."
     },
     {
       "title": "Benchmarking and studying the LLM-based agent system in end-to-end software development",
       "authors": ["Zijie Zeng", "Yifan Li", "Ruiqi Xie"],
       "year": 2025,
       "arxiv_id": "2511.04064",
       "relevance": "E2EDevBench, a concurrent end-to-end SE benchmark compared against in Table 1."
     },
     {
       "title": "NL2Repo-Bench: Towards long-horizon repository generation evaluation of coding agents",
       "authors": ["Jiawei Ding", "Shuai Long", "Chenhao Pu"],
       "year": 2025,
       "arxiv_id": "2512.12730",
       "relevance": "Repository generation benchmark from NL requirements, concurrent work compared in Table 1."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2023,
       "relevance": "Foundational reasoning-acting paradigm used in modern coding agents."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP benchmark for function-level code generation."
     }
   ]
 }