ai-research-survey

scan.json (28395B)

---

 {
   "paper": {
     "title": "ProjectEval: A Benchmark for Programming Agents Automated Evaluation on Project-Level Code Generation",
     "authors": [
       "Kaiyuan Liu",
       "Youcheng Pan",
       "Yang Xiang",
       "Daojing He",
       "Jing Li",
       "Yexing Du",
       "Tianrun Gao"
     ],
     "year": 2025,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2503.07010",
     "doi": "10.48550/arXiv.2503.07010"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Paper states 'Dataset, code and constructed evaluation machine are available at https://github.com/RyanLoil/ProjectEval/' in a footnote on page 1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset (20 tasks with 284 testcases) is released at the same GitHub URL mentioned in the paper."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, conda environment, or dependency specifications in the paper. Only mentions that Selenium and subprocess are used for testing."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the evaluation process conceptually (Section 3.3) but does not provide step-by-step reproduction instructions, commands to run, or a reproducing-results guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 4, 5, and 7 are reported as point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'close-source LLM agents are doing better than open-source ones', 'cascade generation is better than direct generation') based solely on comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw Pass@K percentages and objective indicator scores. No formal effect sizes (Cohen's d, odds ratios, etc.) are provided. The one mention of '2.06% higher' for cascade vs direct does not include baseline context adequate for interpreting magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains only 20 tasks. No justification is given for why 20 tasks is sufficient, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported across experimental runs despite the Pass@K metric implying multiple runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 14+ models across open-source AGI LLMs, closed-source AGI LLMs, code generation LLMs, and a production coding agent (OpenHands), providing extensive baseline comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models: GPT-4o, Gemini 2.0, Phi-4, Llama 3.2, Gemma 2, and OpenHands. These represent the state of the art at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The cascade vs direct generation comparison (RQ2) and three-level input comparison function as ablation studies, isolating the effect of step-by-step vs direct generation and input granularity."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Pass@K plus four objective indicators: Sentence Transformer similarity for checklists, CodeBLEU for skeletons and code, and Levenshtein Distance for parameter values (Table 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated via test suites (Selenium for web, subprocess for console). No human evaluation of the LLM-generated outputs is performed. Human involvement is limited to benchmark construction."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark tasks serve as the test set. No models are tuned or fine-tuned on the benchmark data; all are evaluated zero-shot with temperature 0."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides per-part breakdowns (Checklist, Skeleton, Code, Parameter Values) across all models and input levels. Results are also broken down by cascade vs direct and by input level."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses three typical failure cases: Invalid Output Format, Missing Essential Files, and Omitted Content, with explanations of likely causes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that code LLMs 'have almost no effective results,' that many open-source models score near zero, that OpenHands got lower scores than raw GPT-4o, and that cascade generation can hurt GPT-4o performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about automated evaluation via user interaction simulation, three input levels, and findings about systematic engineering and overall project understanding are supported by the experimental results in Sections 5.1-5.4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-sounding claims: 'cascade generation... mimics the CoT process' causing better performance, and 'asking the LLM agents to generate according to the thought steps we set induces the LLM to tend to activate parameters about natural language.' These are mechanistic claims without adequate causal justification."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Programming Agents Automated Evaluation on Project-Level Code Generation' broadly, and the conclusion says results apply to 'real-world production environments.' However, the benchmark is 20 Python/Django tasks. The limitation section acknowledges Python-only and Django-focused constraints, but the title and abstract do not bound the generalization."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper briefly speculates that GPT-4o's better direct generation might be due to 'activating parameters about natural language' (Section 5.2) but provides no systematic discussion of alternative explanations for the main findings."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures pass rates on 20 Django-heavy tasks with Selenium/subprocess tests and frames this as measuring 'project-level code generation' capability broadly, without discussing whether these 20 tasks are a valid proxy for general project-level programming ability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Closed-source models are specified only by marketing name: 'GPT-4o', 'GPT-3.5-turbo', 'Gemini 1.5 pro', 'Gemini 2.0-flash' without snapshot dates or API versions. Open-source models include parameter counts (e.g., 'Llama-3.1-7B') but no exact checkpoint identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides the actual reasoning prompt template used, and the fill values ({description} and {technical_stack}) are deterministically derived from the benchmark inputs which are fully specified."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B states 'All the models are running under temperature zero with all settings default in their releases.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The main experiments use direct LLM prompting without agentic scaffolding. OpenHands is evaluated as a third-party black-box tool whose internal scaffolding the authors cannot be expected to describe."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 and Figure 2 document the full construction pipeline: NL Prompt sources → LLM generation → human review → canonical solutions → Masker → skeleton. Each step and its purpose are described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitation' provides a dedicated limitations section with four specific bullet points."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: Django as primary tech stack may not align with all project patterns, canonical answer supports only Python, JSON format is not universally compatible with all LLMs, and CPV may not be the only valid parameter values."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the limitation section mentions Python-only and Django-focused constraints, the paper does not explicitly state what the results do NOT show or what populations/settings are excluded. The boundaries are implicit rather than explicitly stated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository includes the benchmark tasks, test suites, canonical solutions, and canonical parameter values."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the full construction process: 7 tasks sourced from SoftwareDev and ProjectDev, rest created originally. LLM-generated components reviewed by humans. Construction cost of $2.95 (GPT-4o) and $420 (human review) reported."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Human annotators were hired from 'a third-party company with contract' with no further detail on annotator qualifications, selection criteria, or potential biases. The 8-page Chinese instruction guideline is mentioned but not included."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 and Section 3.2 document the full pipeline from NL Prompt sources through LLM generation, human review, test suite creation, canonical solution generation, and skeleton masking. Appendix A provides the full version."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This work is supported by the National Science and Technology Innovation 2030 Major program (Grant No. 2024ZD01NL00101).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Harbin Institute of Technology and Pengcheng Laboratory. No authors are affiliated with the LLM companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is a Chinese government research program with no financial interest in which LLM performs best on the benchmark."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models evaluated, despite this being critical for assessing whether models could have seen the benchmark tasks during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "7 of 20 tasks are sourced from SoftwareDev and ProjectDev, both published benchmarks. No discussion of whether the evaluated models may have been trained on these published tasks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SoftwareDev (2024) and ProjectDev (2024) tasks were published before the evaluation. No discussion of whether models trained after publication could have seen these tasks. No contamination analysis performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Human annotators were involved in benchmark construction only, not as research subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 reports 'The total evaluation cost of Pass@5 with GPT-4o is $28.02, average $5.60 for each round.' Section 3.2 reports construction cost of $2.95 (GPT-4o) and $420 (human review)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Only monetary API costs are reported. No GPU hours, hardware specifications, or total wall-clock time for running all model evaluations is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. The paper states temperature 0 (deterministic) but Pass@5 scores differ from Pass@1, suggesting non-determinism that is not investigated."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports Pass@1 and Pass@5 metrics but does not explicitly state how many total samples were generated per task to compute these metrics."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter search was performed. All models use temperature 0 with default settings."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection was performed. All models use default settings."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The authors evaluate third-party LLMs on their benchmark, not their own model against baselines. The Lucic et al. concern about re-implementing baselines does not apply."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 3B-14B parameter open-source models against large closed-source models (GPT-4o, Gemini 1.5 pro) without controlling for or discussing compute budget differences. The finding that closed-source models outperform open-source ones is unsurprising given the compute disparity."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper argues user interaction simulation is better than test units but provides no empirical validation that their 20 Django-heavy tasks with Selenium tests actually measure 'project-level code generation capability' as claimed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "OpenHands (GPT-4o + scaffold) is compared against raw GPT-4o without systematically controlling for the scaffold effect. The paper notes OpenHands scored lower but attributes it to task origin without analyzing the scaffold confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "7 of 20 tasks are sourced from SoftwareDev (2024) and ProjectDev (2024), published before the models' training cutoffs. No temporal leakage analysis is performed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The cascade mode provides increasingly detailed inputs (NL → Checklist → Skeleton) but the paper does not analyze whether this constitutes feature leakage from the canonical solution structure."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 20 tasks are independent or share structural patterns (e.g., Django-heavy tasks may share common patterns that inflate scores for models trained on Django code)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, or decontamination procedures."
       }
     }
   },
   "claims": [
     {
       "claim": "ProjectEval is hard for current LLM agents, with only GPT-4o reaching Pass@5 of approximately 15%.",
       "evidence": "Table 4 shows GPT-4o achieves 12.49% overall average Pass@5, with the next best (Gemini 1.5 pro) at 6.65%. All open-source models are below 1.13%.",
       "supported": "strong"
     },
     {
       "claim": "Closed-source LLM agents perform better than open-source ones on project-level code generation.",
       "evidence": "Tables 4 and 5 consistently show GPT-4o and Gemini models outperforming all open-source models across all metrics and input levels.",
       "supported": "moderate"
     },
     {
       "claim": "Cascade generation (step-by-step) is better than direct generation on average.",
       "evidence": "Table 5 objective indicators show cascade is 2.06% higher at Level 1 input on average. However, Table 4 shows GPT-4o achieves higher Pass@K scores in direct mode, contradicting the general claim.",
       "supported": "weak"
     },
     {
       "claim": "LLM agents are best at generating Checklists compared to other parts of ProjectEval.",
       "evidence": "Table 5 shows checklist scores (29.28% average) are substantially higher than skeleton (7.59%), code (11.34%), and parameter values (11.10%) in cascade generation.",
       "supported": "moderate"
     },
     {
       "claim": "Code generation LLMs (StarCoder-2, CodeGemma, CodeLlama) cannot produce effective project-level code.",
       "evidence": "Table 4 shows all code generation LLMs have near-zero Pass@K scores (0.00-1.20% Pass@5 on Level 3 skeleton only). Table 5 confirms negligible objective indicator scores.",
       "supported": "moderate"
     },
     {
       "claim": "Systematic engineering, overall project understanding, and comprehensive analysis are key capabilities for LLM agents.",
       "evidence": "This is stated as a finding in the abstract and Section 5.3, derived from observing that closed-source models do better across all parts. The claim is an interpretation rather than a directly measured outcome.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ProjectEval introduces a 20-task benchmark for project-level code generation with automated evaluation via user interaction simulation (Selenium/subprocess). The best model (GPT-4o) achieves only ~12.5% Pass@5 overall, demonstrating that project-level code generation remains extremely challenging. Closed-source models substantially outperform open-source ones, and code-specialized LLMs perform worst. Cascade (step-by-step) generation shows mixed results: better on objective similarity metrics but worse for GPT-4o on actual pass rates.",
   "red_flags": [
     {
       "flag": "Very small benchmark size",
       "detail": "Only 20 tasks with 284 total testcases. This is too small to draw robust conclusions about model capabilities, especially for per-model or per-level comparisons. No statistical power analysis or justification for the sample size."
     },
     {
       "flag": "GPT-4o used for construction and evaluation",
       "detail": "GPT-4o was used to generate the benchmark components (canonical solutions, checklists, test suites, parameter descriptions) and is also the best-performing model on the benchmark. This creates a potential bias: the tasks and evaluation criteria may implicitly favor GPT-4o's coding style and patterns."
     },
     {
       "flag": "Non-standard or unclear Pass@K metric",
       "detail": "Pass@5 scores are consistently lower than Pass@1 scores across most models (e.g., GPT-4o Cascade Level 1: Pass@1=10.21, Pass@5=8.52), which contradicts the standard definition where Pass@K >= Pass@1 for K >= 1. The paper claims to follow HumanEval convention but the metric appears to work differently without adequate explanation."
     },
     {
       "flag": "No contamination analysis despite reused tasks",
       "detail": "7 of 20 tasks (35%) are sourced from SoftwareDev and ProjectDev, both publicly published benchmarks. Models trained after these publications may have seen the tasks. No contamination analysis is performed."
     },
     {
       "flag": "No error bars or statistical tests",
       "detail": "All comparative claims are made by directly comparing point estimates without any statistical tests, confidence intervals, or measures of uncertainty. With only 20 tasks, stochastic variation could easily account for observed differences."
     },
     {
       "flag": "Django-heavy benchmark presented as general",
       "detail": "The benchmark is heavily biased toward Django web development (acknowledged in limitations as 'primary technical stack'). Results are presented as measuring general 'project-level code generation' capability, which Django-specific tasks do not represent."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jared Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval, the foundational benchmark for LLM code generation evaluation."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong", "Mingchen Zhuge", "Jonathan Chen"],
       "year": 2024,
       "relevance": "Multi-agent framework for software development; used as a source for ProjectEval tasks and represents SoftwareDev benchmark."
     },
     {
       "title": "AgileCoder: Dynamic collaborative agents for software development based on agile methodology",
       "authors": ["Minh Huynh Nguyen", "Thang Phan Chau", "Phong X. Nguyen", "Nghi D. Q. Bui"],
       "year": 2024,
       "arxiv_id": "2406.11912",
       "relevance": "Multi-agent software development system using agile methodology; represents ProjectDev benchmark used as a source for ProjectEval tasks."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": ["Chen Qian", "Wei Liu", "Hongzhang Liu"],
       "year": 2024,
       "relevance": "LLM-based multi-agent system for collaborative software development via natural language communication."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating LLM code generation on real-world GitHub issue resolution."
     },
     {
       "title": "Prompting large language models to tackle the full software development lifecycle: A case study",
       "authors": ["Bowen Li", "Wenhan Wu", "Ziwei Tang"],
       "year": 2024,
       "arxiv_id": "2403.08604",
       "relevance": "DevBench project-level benchmark with automated test-unit evaluation, the primary comparison point for ProjectEval."
     },
     {
       "title": "OpenHands: An open platform for ai software developers as generalist agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2025,
       "arxiv_id": "2407.16741",
       "relevance": "Production coding agent platform evaluated on ProjectEval; represents state-of-the-art agentic coding systems."
     },
     {
       "title": "Commit0: Library generation from scratch",
       "authors": ["Wenting Zhao", "Nan Jiang", "Celine Lee"],
       "year": 2024,
       "arxiv_id": "2412.01769",
       "relevance": "Benchmark for generating entire libraries from scratch, closely related to project-level code generation evaluation."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP benchmark for LLM program synthesis, foundational work in code generation evaluation."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation",
       "authors": ["Xueying Du", "Mingwei Liu", "Kaixin Wang"],
       "year": 2023,
       "arxiv_id": "2308.01861",
       "relevance": "Class-level code generation benchmark bridging function-level and project-level evaluation granularity."
     },
     {
       "title": "Exploring and evaluating hallucinations in LLM-powered code generation",
       "authors": ["Fang Liu", "Yang Liu", "Lin Shi"],
       "year": 2024,
       "arxiv_id": "2404.00971",
       "relevance": "Studies hallucination in LLM code generation, directly relevant to understanding code quality issues."
     }
   ]
 }