ai-research-survey

scan.json (22883B)

---

 {
   "paper": {
     "title": "Benchmarking AI Models in Software Engineering: A Review, Search Tool, and Unified Approach for Elevating Benchmark Quality",
     "authors": ["Roham Koohestani", "Philippe de Bekker", "Begüm Koç", "Maliheh Izadi"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.05860"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided at https://github.com/AISE-TUDelft/AI4SE-benchmarks. The paper states 'we publicly release the material of our review, user study, and the enhanced benchmark.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The review data, user study data, and 50% of the manually refined HumanEvalNext benchmark are publicly released via the GitHub repository. The paper states full benchmark will be available upon acceptance."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'one NVIDIA A100 80GB GPU and 32 CPU cores' for inference but does not provide requirements.txt, Dockerfile, or detailed library versions for reproducing the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The experimental setup section describes the general approach but lacks specific commands or scripts to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Pass@1 results are reported as point estimates without confidence intervals or error bars. The boxplot in Figure 7 shows distributions but individual model results lack uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The agentic pipeline noninferiority study uses one-sided one-sample t-tests and Wilcoxon signed-rank tests with reported p-values. However, no significance tests are applied to the main HumanEval vs HumanEvalNext performance comparison claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance drops are reported with baseline context: e.g., 'average pass@1 score decreases by 31.2%' from specific baselines (e.g., Nxcode-CQ from 87.23% to 51.22%). Mean ratings with SD reported for noninferiority study (0.16, SD=0.67)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the choice of 10 models, 22 user study participants, or 100 MBPP problems for the generalizability study. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported for the noninferiority study (SD=0.67, 0.51, 0.49). The boxplot in Figure 7 shows the distribution of pass@1 drops across models."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "HumanEvalNext is compared against the original HumanEval and HumanEvalPlus benchmarks. The agentic pipeline is compared against the human-improved version."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The 10 evaluated models are mostly 6-15B parameter open-source models. No frontier models (GPT-4, Claude, Gemini) are evaluated, and the paper acknowledges this gap in the discussion section."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study isolates the contribution of individual BenchFrame improvements (corrected solutions, added type annotations, improved tests, edge cases). All changes are bundled together."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only pass@1 is used as the evaluation metric for model performance on the benchmarks. No other metrics (e.g., pass@5, pass@10, execution time) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The user study with 22 participants evaluated BenchScout. Two independent reviewers rated human vs. agentic benchmark improvements on a 5-point ordinal scale for the noninferiority study."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning training study; the benchmarks are evaluation instruments, not train/test splits for model development."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Per-model breakdowns are provided in Tables XXII-XXV. Per-category distribution of benchmarks is shown in Figure 3. Per-problem difficulty ranking is discussed."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific failure cases: CodeQwen1.5 dropping from 87.2% to 10.98%, specific HumanEval Task 47 errors, and pitfalls of the agentic pipeline (docstring leaking solutions, unintended assumptions)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The agentic pipeline's pitfalls are reported (docstrings revealing solutions, canonical solutions making unintended assumptions, test format issues). The paper also shows the agentic version sometimes performs worse than human version (Table XXIV)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 31.22% and 19.94% pass@1 drops are supported by Tables XXII-XXIII. User study scores of 4.5, 4.0, and 4.1 are supported by the questionnaire results. 273 benchmarks from 247 studies is supported by Section III."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implies that performance drops are caused by better benchmark quality ('enhanced difficulty and refined evaluation precision'), but confounds exist: added edge cases, corrected errors, and increased test count are not isolated. The dramatic CodeQwen1.5 drop (87.2% to 10.98%) is attributed to data leakage without direct evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper proposes BenchFrame as a general methodology but validates it only on HumanEval and 100 MBPP problems, both Python-only code generation tasks. Claims like 'one can extend it to other datasets with similar success' go beyond the evidence. The title claims 'Benchmarking AI Models in Software Engineering' broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section discusses generic methodological concerns (selection bias, generalizability) but does not discuss specific alternative explanations for the performance drops, such as whether test case inflation rather than quality improvement drives the results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Full HuggingFace model identifiers are specified (e.g., 'NTQAI/Nxcode-CQ-7B-orpo', 'deepseek-ai/deepseek-coder-6.7b-instruct'). The agentic pipeline uses 'o3-mini-2025-01-31' with specific date."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the prompting approach ('an instructional preamble asking the model to finish the implementation') but does not provide the actual prompt text used for evaluation."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or other sampling parameters are reported for the model evaluations. A 15-second timeout is mentioned but core inference parameters are absent."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic pipeline for BenchFrame is described in detail with a flowchart (Figure 8) showing the three phases (text, code, test improvement), validation loop, retry logic (3 attempts), and fallback to minimal passing tests."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The literature review pipeline is documented: search platforms (Google Scholar, Semantic Scholar), keywords, selection criteria (originality, reproducibility, accessibility), duplicate removal, forward/backward snowballing, resulting in 247 papers. Inter-rater agreement of 96.4% is reported."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-E 'Threats to the Validity' discusses construct, internal, and external validity threats in detail."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: the 22-participant sample may limit generalizability, 10 models may be insufficient, HumanEvalNext will also be affected by data leakage over time, the user study may not capture full tool functionality."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. While limitations mention possible extensions (more models, more languages), specific scope boundaries (e.g., 'these results apply only to Python function-level code generation benchmarks') are not stated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at https://github.com/AISE-TUDelft/AI4SE-benchmarks includes review data, user study data, and 50% of HumanEvalNext. Full benchmark promised upon acceptance."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes the review methodology: search platforms, keywords, selection criteria, duplicate removal, snowballing, taxonomy development. Section IV-C describes user study recruitment."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "User study participants are described: 22 people from industry (9) and academia (13), with roles (6 Researchers, 5 PhD Candidates, 5 Students, 4 Engineers, 2 Lead Researchers) and experience levels detailed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The review pipeline documents stages from search to final benchmark list. The BenchScout pipeline (Figure 5) shows data collection through metadata enrichment, clustering, and frontend. The agentic pipeline (Figure 8) shows the benchmark improvement flow."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section IX states: 'This research was supported in part by an Amazon Research Award granted to Dr. Maliheh Izadi.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with EEMCS faculty, Delft University of Technology, The Netherlands."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Amazon funded the research but the paper does not evaluate any Amazon products. The acknowledgment states 'The views and conclusions contained in this paper are those of the authors and do not necessarily reflect the position or policies of Amazon.'"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper beyond the funding acknowledgment."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 10 code LLMs on HumanEval benchmarks but does not state the training data cutoff dates for any of the models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses data contamination: HumanEval Task 47 shows ChatGPT reproducing benchmark errors (Figure 1), suggesting contamination. The paper argues HumanEvalNext mitigates this by introducing new edge cases and modifications."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is a central theme. The paper shows evidence of contamination (ChatGPT reproducing HumanEval errors), discusses how CodeQwen1.5's dramatic drop suggests data leakage, and positions HumanEvalNext as a mitigation strategy."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the user study with 22 participants."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the user study."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Demographics are detailed: roles (Researchers, PhD Candidates, Students, Engineers, Lead Researchers), experience levels (1-3, 3-5, 5+, <1 years), industry (9) vs academia (13), and AI4SE familiarity ratings."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "The paper states the aim to 'assess the tool's effectiveness across a diverse group of users with varying degrees of expertise' but does not specify inclusion/exclusion criteria for participant selection."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The user study is not an experimental study with treatment/control conditions; all participants used the same tool."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "For the peer review of HumanEvalNext: 'the reviewer was not informed of specific changes made by the first author.' However, the user study itself had no blinding."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of whether all 22 participants completed the full study or if any dropped out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The agentic pipeline cost is reported: 'the total incurred cost from calling the apis for the models was $5.2823.' Manual effort is also quantified: 'initial creation took over 100 hours, the independent peer-review process required an additional 16 hours.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware is stated: 'one NVIDIA A100 80GB GPU and 32 CPU cores.' The manual effort (100+ hours for benchmark creation, 16 hours for peer review) is quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "Average pass@1 scores decrease by 31.22% (mean) and 26.02% (median) when evaluating 10 LLMs on HumanEvalNext compared to original HumanEval.",
       "evidence": "Table XXIII shows per-model pass@1 scores on HumanEval vs HumanEvalNext, with drops ranging from 18.94% to 76.22%. Figure 7 shows the distribution.",
       "supported": "strong"
     },
     {
       "claim": "273 AI4SE benchmarks were identified from 247 studies since 2014.",
       "evidence": "Section III-A describes the systematic review methodology with search criteria, quality assessment, and snowballing. Figure 2 shows publication trends.",
       "supported": "strong"
     },
     {
       "claim": "BenchScout achieved usability, effectiveness, and intuitiveness scores of 4.5, 4.0, and 4.1 out of 5 in a user study with 22 participants.",
       "evidence": "Section IV-C reports detailed questionnaire results with demographic breakdown, Likert scale responses, and qualitative feedback.",
       "supported": "moderate"
     },
     {
       "claim": "The agentic pipeline is noninferior to the human-improved process for benchmark refinement.",
       "evidence": "Noninferiority tests with margin δ=-0.5: mean ratings 0.16-0.53, t-statistics 12.68-25.75, p-values all 1.000, Wilcoxon tests confirming (Section VI-B).",
       "supported": "strong"
     },
     {
       "claim": "Certain models may have benefited from data leakage on the original HumanEval, as evidenced by CodeQwen1.5 dropping from 87.2% to 10.98%.",
       "evidence": "Table XXIII shows the dramatic drop. The ChatGPT reproducing HumanEval Task 47 errors (Figure 1) provides indirect evidence. But no direct contamination analysis is performed.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["meta-analysis", "benchmark-eval", "case-study"],
   "key_findings": "This paper reviews 273 AI4SE benchmarks from 247 studies, revealing a fragmented landscape dominated by code generation (34.4%) with systemic issues including poor maintenance, language specificity, and lack of peer review. BenchFrame, a peer-review-oriented methodology for improving benchmark quality, is demonstrated through HumanEvalNext, which causes an average 31.2% drop in pass@1 scores across 10 code LLMs compared to original HumanEval. An agentic pipeline for automating benchmark improvements is shown to be noninferior to manual human refinement, with API costs of only $5.28.",
   "red_flags": [
     {
       "flag": "No ablation of individual improvements",
       "detail": "BenchFrame bundles multiple changes (corrected solutions, type annotations, more tests, edge cases, clearer descriptions) without isolating which changes cause the performance drops. The 31.2% decline could be driven primarily by increased test count (1.92x more asserts) rather than quality improvements."
     },
     {
       "flag": "Single metric evaluation",
       "detail": "Only pass@1 is reported for all model evaluations. No other metrics (pass@5, pass@10, execution efficiency) are used, limiting the robustness of performance claims."
     },
     {
       "flag": "Weak causal claim for data leakage",
       "detail": "CodeQwen1.5's dramatic 76.22% drop is attributed to data leakage without direct evidence of contamination. Alternative explanations (e.g., the model being particularly sensitive to edge cases or type annotations) are not explored."
     },
     {
       "flag": "Missing hyperparameters for model evaluations",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for the 10 model evaluations, making reproduction difficult and potentially affecting pass@1 results significantly."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Original HumanEval benchmark paper, foundational to code generation evaluation."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "HumanEvalPlus/EvalPlus benchmark that augments HumanEval test suites, directly compared in this study."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["N. Jain"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Dynamic benchmark approach addressing contamination, positioned as complementary to BenchFrame's repair-and-refine approach."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["C. E. Jimenez"],
       "year": 2024,
       "relevance": "Major real-world SE benchmark for evaluating LLM agents on GitHub issue resolution."
     },
     {
       "title": "BigCodeBench",
       "authors": ["T. Y. Zhuo"],
       "year": 2024,
       "relevance": "Large-scale code generation benchmark with 1,140 Python problems, representative of class-level evaluation."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["X. Liu"],
       "year": 2023,
       "relevance": "Benchmark for evaluating LLMs as agents across multiple environments."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["J. Austin"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "MBPP benchmark paper, one of the two foundational benchmarks improved by BenchFrame in this study."
     },
     {
       "title": "Top leaderboard ranking = top coding proficiency, always? EvoEval: Evolving coding benchmarks via LLM",
       "authors": ["C. S. Xia", "Y. Deng", "L. Zhang"],
       "year": 2024,
       "arxiv_id": "2403.19114",
       "relevance": "Addresses benchmark evolution and leaderboard gaming in code generation evaluation."
     },
     {
       "title": "CodeElo: Benchmarking Competition-level Code Generation of LLMs with Human-comparable Elo Ratings",
       "authors": ["S. Quan"],
       "year": 2025,
       "arxiv_id": "2501.01257",
       "relevance": "Competition-level code generation benchmark with Elo-based rating system."
     },
     {
       "title": "CodeRAG-Bench",
       "authors": ["Z. Wang"],
       "year": 2024,
       "relevance": "Benchmark for retrieval-augmented code generation evaluation."
     }
   ]
 }