ai-research-survey

scan.json (28616B)

---

 {
   "paper": {
     "title": "Benchmarking AI Models in Software Engineering: A Review, Search Tool, and Unified Approach for Elevating Benchmark Quality",
     "authors": [
       "Roham Koohestani",
       "Philippe de Bekker",
       "Begüm Koç",
       "Maliheh Izadi"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.05860"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository (https://github.com/AISE-TUDelft/AI4SE-benchmarks) and the BenchScout tool at https://evalpro.online/. Referenced as a replication package multiple times (Sections I, III-A, IV-C)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII-A states: 'We publicly release the results of our literature review, user study, and 50% of the manually refined benchmark.' The full benchmark is promised upon acceptance. The review data and BenchScout data are available via the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section V-B mentions 'one NVIDIA A100 80GB GPU and 32 CPU cores' for inference, but no software environment details (requirements.txt, Dockerfile, Python version, library versions) are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a replication package is referenced via GitHub, the paper itself does not contain step-by-step reproduction instructions. The experimental setup in Section V-B describes the general approach but lacks specific commands or scripts to replicate results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark evaluation results (Tables XXII-XXV) report only point estimates of pass@1 scores with no confidence intervals or error bars. The user study reports only average Likert scores without uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-B reports t-tests and Wilcoxon signed-rank tests for the noninferiority comparison between HumanEvalNext and HumanEvalNext-Agentic, with specific p-values (e.g., p = 3.46 × 10^−20). However, no significance tests are applied to the main benchmark comparison (HumanEval vs. HumanEvalNext) or user study results."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute percentage drops in pass@1 scores with baseline context: '31.22% average decrease' from specific baselines (e.g., 87.23% to 51.22%), and for the noninferiority study reports mean ratings (0.16, 0.53, 0.35) with SDs (0.67, 0.51, 0.49)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for key sample sizes: why 10 models were selected, why 22 user study participants, why 100 MBPP problems for generalizability, or why 2 reviewers for the noninferiority study. No power analysis is mentioned."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark evaluations report single-run results only. No variance, standard deviation, or results across multiple runs are reported for the pass@1 experiments. The noninferiority study reports SD for reviewer ratings, but the main experiments do not report any spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The HumanEvalNext benchmark is compared against both the original HumanEval and HumanEvalPlus (Table XXII). The agentic pipeline is compared to the human-improved version. MBPPNext is compared to MBPP baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The 10 models evaluated are all open-source models from leaderboards 'at the evaluation's start' (Section V-B), but they are all relatively small (3B-15B) and older models. No frontier models (GPT-4, Claude, Gemini) are included. The paper itself acknowledges in Section VI-D that 'it remains unclear how larger, top-performing models behind paywalls, such as GPT and Gemini, would perform.'"
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed to determine which specific improvements in BenchFrame (corrected solutions, type annotations, edge cases, improved descriptions) contribute most to the performance drops. The modifications are bundled together."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark evaluation uses only pass@1 as the metric. No other metrics (pass@k for k>1, functional correctness breakdowns, etc.) are reported for the code generation experiments. The user study uses multiple Likert scale questions but the code generation evaluation is single-metric."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The user study with 22 participants evaluates BenchScout (Section IV-C). The peer review of HumanEvalNext (Section V-A3) and the paired evaluation with two independent reviewers for the agentic pipeline (Section VI-B) both involve human evaluation of the benchmark quality."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark creation and review paper, not a machine learning training paper. There is no train/test split relevant to the paper's own methodology."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table XXII/XXIII provides per-model breakdowns. The review provides per-category distribution (Figure 3: code generation 34.4%, code understanding 17.7%, etc.). Table XIX provides per-limitation frequency counts. Per-problem difficulty analysis is also discussed."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-B discusses pitfalls of the agentic pipeline: docstrings revealing solutions, canonical solutions making unintended assumptions, test cases not adhering to format. Section V-A1 discusses specific failure modes in HumanEval (incorrect tests, suboptimal solutions). The CodeQwen1.5 extreme drop (87.2% to 10.98%) is discussed as indicating potential data leakage."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-B reports negative observations about the agentic pipeline (docstring leakage, unintended assumptions in solutions). Table XXIV shows the agentic version sometimes performs worse than human-improved version on specific models. The paper honestly discusses limitations of BenchScout's visualization features receiving lower scores (3.7-3.9)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: '247 studies, identifying 273 AI4SE benchmarks' matches Section III-B; BenchScout scores of 4.5, 4.0, 4.1 match Section IV-C; pass@1 drops of 31.22% and 19.94% match Section V-C (Tables XXII-XXIII). All claims are grounded in reported results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as CodeQwen's drop 'suggests that certain models may have benefited from data leakage' (Section V-C) and that BenchFrame 'reveals substantial performance gaps' (Section VIII). These are correlational observations attributed to causal mechanisms (benchmark flaws, contamination) without controlled experiments isolating these factors. The bundled modifications prevent attributing drops to specific causes."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to be about 'Benchmarking AI Models in Software Engineering' broadly, but the evaluation is limited to Python code generation with 10 small open-source models. The MBPP generalizability test uses only 100 of 500 problems. The paper's framing as a 'unified approach' for benchmark quality extends well beyond what was tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C discusses data leakage as an alternative explanation for model performance patterns. Section VI-E (Threats to Validity) discusses construct validity concerns about subjectivity in inclusion criteria, internal validity concerns about selection bias in the user study, and external validity concerns about generalizability limitations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B lists exact model identifiers for all 10 models (e.g., 'NTQAI/Nxcode-CQ-7B-orpo', 'deepseek-ai/deepseek-coder-6.7b-instruct'). Section VI-B specifies 'o3-mini-2025-01-31' for the agentic pipeline. These are HuggingFace model IDs that identify specific versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section V-B describes the prompting approach only in natural language: 'the LLM is prompted using an instructional preamble asking the model to finish the implementation of the function.' The actual prompt text is not provided in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported: no temperature, top-p, max tokens, or sampling settings for any of the model inference runs. Section V-B mentions only GPU type (A100 80GB) and timeout (15 seconds) but not LLM generation parameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic pipeline for BenchFrame is described in detail in Section VI-B with a flowchart (Figure 8) showing three phases (text improvement, code improvement, test improvement), validation, retry logic (3 attempts), and minimized passing test cases fallback. The model used (o3-mini-2025-01-31) and the OpenAI API are specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes the literature review pipeline: structured searches on Google Scholar and Semantic Scholar with specific keywords, credibility verification, taxonomy development, duplicate removal, forward/backward snowballing, resulting in 247 papers. Inter-rater agreement (96.4%) is reported for categorization."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-E 'Threats to the Validity' provides a dedicated subsection covering construct validity, internal validity, and external validity, with substantive discussion of specific threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: 'subjectivity in defining and applying the inclusion/exclusion criteria' for the review, 'selection bias' in the user study mitigated by including participants from both industry and academia, the reviewer 'was not informed of specific changes made by the first author' for blinding, and acknowledgment that '10 models... could not be considered sufficient to prove generalizability.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While specific threats are discussed, the paper does not explicitly state what the results do NOT show. It does not bound claims to Python-only or small open-source models. The title and framing suggest broad applicability ('Benchmarking AI Models in Software Engineering') without explicit statements about excluded settings, languages, or model classes."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at https://github.com/AISE-TUDelft/AI4SE-benchmarks is referenced multiple times. Section VIII-A confirms public release of review data, user study details, and 50% of HumanEvalNext. The benchmark data and metadata are available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A details the data collection: searches on Google Scholar and Semantic Scholar using specific keywords, PapersWithCode datasets collection, English papers from 2014-2025, two authors reviewing relevance, forward and backward snowballing, 247 papers total."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the user study with 22 participants, Section IV-C describes demographics (9 industry, 13 academia, roles, experience levels) but does not describe how participants were recruited. No information on recruitment channels, compensation, or potential selection bias in participant recruitment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The review pipeline is documented from search to final analysis: keyword search → duplicate removal → relevance review by two authors → originality/reproducibility/accessibility assessment → snowballing → 247 papers → taxonomy development → metadata extraction. The BenchFrame pipeline is documented in Figure 6 and Section V-A."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section IX (Acknowledgments) states: 'This research was supported in part by an Amazon Research Award granted to Dr. Maliheh Izadi.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with 'EEMCS faculty, Delft University of Technology, The Netherlands.' ORCID identifiers are provided for three authors. No author works at a company whose product is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is Amazon, which does not have a direct commercial stake in the specific benchmarks being evaluated (HumanEval, MBPP). The paper states: 'The views and conclusions contained in this paper are those of the authors and do not necessarily reflect the position or policies of Amazon.'"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The acknowledgments mention the Amazon grant but there is no explicit declaration that the authors have no other financial interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 10 pre-trained code models on HumanEval/HumanEvalNext benchmarks but does not state the training data cutoff dates for any of the models. This is important given that the paper itself discusses data leakage as a concern."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses potential data contamination: Section I discusses HumanEval contamination with ChatGPT reproducing incorrect answers, Section V-C discusses 'data leakage or other issues in the original HumanEval benchmark', and Section VI-D acknowledges HumanEvalNext would also be affected by data leakage over time."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark contamination is a central theme of the paper. Section I discusses ChatGPT reproducing HumanEval errors as evidence of contamination. Section V-C interprets the large performance drops as evidence of prior benchmark overfitting. Section VII discusses LiveCodeBench as an approach to avoid overfitting and DyCodeEval for minimizing contamination."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The paper includes a user study with 22 participants but does not mention pre-registration of the study design or hypotheses."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics board approval for the user study involving 22 human participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C reports participant demographics: roles (6 researchers, 5 PhD candidates, 5 students, 4 engineers, 2 lead researchers), experience levels (8 at 1-3 years, 6 at 3-5, 3 at 5+, 5 at <1 year), industry vs academia split (9 vs 13), and AI4SE familiarity ratings."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "Section IV-C describes aiming for 'a diverse group of users with varying degrees of expertise' and includes people from 'both industry and academia' but does not state specific inclusion or exclusion criteria for participant selection."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The user study is not an experimental study with treatment/control conditions requiring randomization. All participants used the same tool and filled out the same questionnaire."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "The user study is not an experimental study with multiple conditions where blinding would be applicable. It is a single-tool usability evaluation."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information is provided about whether all 22 participants completed the study or whether any dropped out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-B reports the API cost for the agentic pipeline: 'the total incurred cost from calling the apis for the models was $5.2823.' However, no cost is reported for the main benchmark inference runs with the 10 models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B states: 'We run the inference for the models on a cluster with one NVIDIA A100 80GB GPU and 32 CPU cores.' Section V-A3 mentions 'the initial creation of the benchmark took over 100 hours' with '16 hours' for peer review. However, total GPU hours for inference are not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "The AI4SE benchmarking landscape is highly fragmented, with 273 benchmarks identified from 247 studies since 2014.",
       "evidence": "Section III-B reports the systematic review results with 273 benchmarks, 71 published in 2024 alone. Figure 2 shows temporal growth. Section III-C identifies systemic limitations including language specificity, poor maintenance, and lack of peer review.",
       "supported": "strong"
     },
     {
       "claim": "BenchScout achieved usability, effectiveness, and intuitiveness scores of 4.5, 4.0, and 4.1 out of 5 in a user study with 22 participants.",
       "evidence": "Section IV-C reports these exact scores from 22 participants (9 industry, 13 academia) using a 5-point Likert scale questionnaire. Detailed demographic and per-question breakdowns are provided.",
       "supported": "moderate"
     },
     {
       "claim": "HumanEvalNext reveals a 31.22% average drop in pass@1 scores compared to the original HumanEval across 10 state-of-the-art code models.",
       "evidence": "Table XXII/XXIII provides per-model pass@1 comparisons. Drops range from 18.94% (codegemma) to 76.22% (CodeQwen1.5). Section V-C reports 31.22% average and 26.02% median decline.",
       "supported": "moderate"
     },
     {
       "claim": "The agentic pipeline for benchmark improvement is noninferior to the human-improved process.",
       "evidence": "Section VI-B reports noninferiority testing with δ=-0.5 margin: mean ratings 0.16 (SD=0.67) for Reviewer 1, 0.53 (SD=0.51) for Reviewer 2, with t-statistics 12.68 and 25.75 and Wilcoxon tests confirming (p < 10^-20).",
       "supported": "moderate"
     },
     {
       "claim": "Some models' performance drops suggest potential data leakage or benchmark overfitting, as model rankings change substantially between HumanEval and HumanEvalNext.",
       "evidence": "Section V-C: CodeQwen1.5 drops from 87.2% to 10.98%, while deepseek-coder maintains relative performance (80.22% to 58.54%). This ranking change is presented as evidence but is observational without controlled contamination analysis.",
       "supported": "weak"
     },
     {
       "claim": "BenchFrame generalizes to MBPP, with an average decrease of 13.4 percentage points on MBPPNext.",
       "evidence": "Section VI-C and Table XXV show per-model results for 100 MBPP problems processed through BenchFrame. Drops range from 6% to 22%.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "meta-analysis",
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "This paper conducts a systematic review of 273 AI4SE benchmarks from 247 studies, revealing fragmentation and quality issues including language specificity, poor maintenance, and lack of peer review. The authors introduce BenchScout, a semantic search tool for benchmark discovery, and BenchFrame, a methodology for improving benchmark quality. Applying BenchFrame to HumanEval yielded HumanEvalNext, which reduced average pass@1 scores by 31.22% across 10 models, suggesting widespread benchmark saturation and potential contamination. An agentic pipeline was shown to be noninferior to the manual improvement process, costing only $5.28 in API calls.",
   "red_flags": [
     {
       "flag": "No hyperparameters reported for main experiments",
       "detail": "The paper evaluates 10 code generation models without reporting temperature, top-p, or other sampling parameters. These settings significantly affect pass@1 results and could explain some of the observed variance between models."
     },
     {
       "flag": "Single-run results without variance",
       "detail": "All pass@1 results appear to be from single runs. Code generation with LLMs involves stochastic sampling, so results without variance across multiple runs or seeds cannot be reliably compared."
     },
     {
       "flag": "Bundled modifications prevent attribution",
       "detail": "BenchFrame modifies multiple aspects simultaneously (corrected solutions, type annotations, edge cases, improved descriptions, increased test count). Without ablation, it is impossible to determine which changes drive the performance drops."
     },
     {
       "flag": "Overly broad title relative to evaluation scope",
       "detail": "The title suggests comprehensive 'Benchmarking AI Models in Software Engineering' but evaluation is limited to Python code generation with 10 small open-source models (3B-15B). No frontier models, no multi-language evaluation, and limited task diversity."
     },
     {
       "flag": "Prompts not provided",
       "detail": "The actual prompts used for model inference are described only in natural language ('instructional preamble asking the model to finish the implementation'). This prevents faithful reproduction, especially given the paper's own emphasis on benchmark rigor."
     },
     {
       "flag": "Only 2 reviewers for noninferiority study",
       "detail": "The agentic pipeline noninferiority comparison uses only 2 reviewers with a 5-point scale. Inter-rater reliability is not reported, and with only 2 raters, individual reviewer biases could substantially affect conclusions."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen et al."],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval, the most widely-used code generation benchmark and the primary subject of this paper's improvement methodology."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Introduced HumanEvalPlus/EvalPlus with improved test coverage, one of the baselines compared against HumanEvalNext."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["N. Jain et al."],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Dynamic benchmark designed to avoid contamination through continuously gathered new challenges, representing the 'build anew' approach contrasted with BenchFrame."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["C. E. Jimenez et al."],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major real-world software engineering benchmark for evaluating LLMs on repository-level tasks, catalogued in this review."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["X. Liu et al."],
       "year": 2023,
       "arxiv_id": "2308.03688",
       "relevance": "Benchmark for evaluating LLMs as agents across multiple tasks, relevant to the agentic AI evaluation landscape."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["T. Y. Zhuo et al."],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Class-level code generation benchmark with diverse function calls, representing more realistic evaluation beyond HumanEval-style problems."
     },
     {
       "title": "Top leaderboard ranking = top coding proficiency, always? EvoEval: Evolving coding benchmarks via LLM",
       "authors": ["C. S. Xia", "Y. Deng", "L. Zhang"],
       "year": 2024,
       "arxiv_id": "2403.19114",
       "relevance": "Evolving benchmarks using LLMs to address saturation, directly relevant to benchmark quality improvement methodology."
     },
     {
       "title": "MultiPL-E: A scalable and extensible approach to benchmarking neural code generation",
       "authors": ["F. Cassano et al."],
       "year": 2022,
       "arxiv_id": "2208.08227",
       "relevance": "Multi-language code generation benchmark framework, used as the language conversion framework for HumanEvalNext."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["J. Austin et al."],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP benchmark, the second major benchmark subject to BenchFrame improvement in this paper."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Y. Li et al."],
       "year": 2022,
       "doi": "10.1126/science.abq1158",
       "relevance": "Competitive programming code generation benchmark (CodeContests), representative of high-difficulty evaluation in AI4SE."
     }
   ]
 }