ai-research-survey

scan.json (24754B)

---

 {
   "paper": {
     "title": "SWE-Bench+: Enhanced Coding Benchmark for LLMs",
     "authors": ["Reem Aleithan", "Haoran Xue", "Mohammad Mahdi Mohajer", "Elijah Nnorom", "Gias Uddin", "Song Wang"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2410.06992"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The SWE-bench+ dataset is released on Zenodo (https://zenodo.org/records/13879453), referenced in the 'Artifacts' paragraph of Section 3."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The SWE-bench+ dataset of 548 task instances is released on Zenodo for replication and extension."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the methodology at a high level ('we followed the same data collection methodology outlined in the SWE-Bench study') but provides no step-by-step reproduction instructions or scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All resolution rates are reported as point estimates (e.g., 12.47%, 3.97%, 0.55%) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims significant drops in resolution rates (e.g., 12.47% to 3.97%) but uses no statistical tests to determine whether differences are significant."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Resolution rate drops are reported with baseline context: e.g., 'drops from 12.47% to 3.97%' and 'from 18.83% to 3.83% for AutoCodeRover', providing magnitude of change from explicit baselines."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 251 resolved patches and 548 SWE-bench+ instances are used without justification for whether these sample sizes are adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experimental results. Single-run numbers only."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four models are compared: SWE-Agent+GPT-4, SWE-RAG+GPT-4, SWE-RAG+GPT-3.5, and AutoCodeRover+GPT-4o. Original SWE-bench leaderboard numbers serve as baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges that 'other top approaches (e.g., Honeycomb, Amazon Q Developer Agent, and Factory Code Droid) were either closed-sourced commercial tools or not verified.' The evaluated models were at the top during the study period but are not the most competitive approaches available."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark quality analysis, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports resolution rate, effectiveness-aware cost per issue fixed, average cost per instance, and average time per instance (Table 4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Three authors independently performed manual patch validation comparing gold patches to model-generated patches. Disagreements were resolved through discussion (Section 2)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "SWE-bench+ is constructed from issues created after model training cutoffs (post October 2023), serving as a temporally held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down the 251 patches into 6 patterns. Table 2 breaks down patterns for Lite and Verified. Table 3 breaks down per model on SWE-bench+. Figure 7 shows per-project distribution."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Sections 2.1.1 discusses four failure patterns with specific examples (Figures 3-6): solution leak, incorrect fixes, different files changed, and incomplete fixes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The main finding is negative: resolution rates drop dramatically when problematic instances are filtered. This is inherently a negative-results paper about SWE-bench quality."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 32.67% solution leakage, 31.08% weak tests, and the drop from 12.47% to 3.97% are all supported by Table 1 and Figure 1b. The 0.55% SWE-bench+ rate is supported by Section 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims (solution leakage causes inflated resolution rates) supported by a reasonable design: manually classifying patches and showing the rate drops after filtering. The causal mechanism (models copy leaked solutions) is demonstrated with concrete examples."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title frames this as evaluating 'LLMs' generally, but only 4 model configurations (GPT-3.5, GPT-4, GPT-4o with specific scaffolds) are tested. Claims about 'LLMs' are not bounded to the tested models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the performance drops. For example, SWE-bench+ uses different repositories/time periods which could introduce difficulty confounds beyond just removing leakage."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly addresses the gap between what SWE-bench measures (test pass rate) and what it claims (issue resolution capability). The core contribution is showing this proxy is unreliable due to weak tests and leaked solutions."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: 'GPT-3.5 (turbo-16k-0613)', 'GPT-4 (1106)', 'GPT-4o (2024-05-13)', and 'AutoCodeRover (v20240620)'."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates existing tools (SWE-Agent, SWE-RAG, AutoCodeRover) following their published instructions. The authors did not design custom prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for any of the models used."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates third-party tools (SWE-Agent, AutoCodeRover) as black boxes, following their published instructions. It cannot be expected to describe their internal scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the data collection pipeline: same 12 projects minus Django, temporal filter (post 2023-11-01), attribute filtering, execution filter, manual screening for solution leakage. Steps with counts are provided (548 final instances)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion mentions 'weak test cases continue to pose challenges' but this is a finding, not a limitation of the study itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential biases in the manual patch classification or limitations of using only 4 model configurations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. No discussion of scope limitations regarding the models tested, the Python-only focus, or the generalizability of findings to other benchmarks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The SWE-bench+ dataset is released on Zenodo (https://zenodo.org/records/13879453), enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the collection: same 12 projects as SWE-bench (minus Django), issues from 2023-11-01 to 2024-08-22, same filtering methodology as original SWE-bench, manual screening for solution leakage."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is GitHub issues from a standard set of repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the 4-step evaluation pipeline: Step 1 store patches, Step 2 evaluate with SWE-bench scripts, Step 3 filter resolved instances, Step 4 manual patch validation. The collection pipeline in Section 3 also includes filtering stages."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are from York University's Lassonde School of Engineering, clearly stated in the header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Training cutoffs are explicitly stated: GPT-3.5 September 2021, GPT-4 April 2023, GPT-4o October 2023 (Section 3)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core contribution addresses this: '94% of the instances in SWE-bench and their pull requests were created prior to the training cut-off dates of the LLMs' (Section 1). SWE-bench+ uses temporal splits to address this."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly addresses contamination by constructing SWE-bench+ with issues from after October 2023, ensuring temporal separation from all model training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark quality analysis study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports average cost per instance and effectiveness-aware cost per issue for all four models (e.g., SWE-Agent+GPT-4: $0.24/instance, $655/fix)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports average time per instance. Total compute is derivable: SWE-Agent+GPT-4 ~37 hours, AutoCodeRover ~41 hours for all 548 instances."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple runs or seed sensitivity. All results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results appear to be from a single execution of each model."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates existing tools with their default configurations; no hyperparameter search is performed."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection is performed. Models are run with their published default settings."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors build SWE-bench+ and then evaluate models on it. They do not discuss potential bias from being both benchmark creators and evaluators."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 and Table 4 explicitly compare cost vs performance across models, noting trade-offs (e.g., SWE-Agent most expensive but similar performance to cheaper RAG+GPT-4)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about benchmark construct validity — whether SWE-bench actually measures what it claims (issue resolution capability vs. test pass rate with leaked solutions)."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares SWE-Agent, SWE-RAG, and AutoCodeRover — different scaffolds paired with different models — without addressing the scaffolding confound. Performance differences are attributed to model+scaffold bundles without disentangling them."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "This is the paper's primary contribution. They show 94% of SWE-bench issues predate model training cutoffs and construct SWE-bench+ with post-cutoff issues."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies and addresses solution leakage (32.67% of resolved instances had solutions in issue descriptions/comments), which is a form of feature leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether issues within the same repository or by the same contributors might be non-independent or share structural patterns."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Manual inspection by three authors comparing gold patches to issue descriptions/comments to detect solution leakage. Temporal filtering used to prevent data leakage."
       }
     }
   },
   "claims": [
     {
       "claim": "32.67% of SWE-Agent+GPT-4 successful patches involve solution leakage where solutions were directly provided in issue reports or comments.",
       "evidence": "Table 1: 82 of 251 patches classified as solution leak through manual review by three authors (Section 2.1.1).",
       "supported": "strong"
     },
     {
       "claim": "31.08% of passed patches are suspicious due to weak test cases that fail to catch incorrect, incomplete, or wrong-file patches.",
       "evidence": "Table 1: 32 incorrect fixes + 9 different files + 37 incomplete fixes = 78 of 251 patches (Section 2.1.1).",
       "supported": "strong"
     },
     {
       "claim": "After filtering problematic instances, SWE-Agent+GPT-4 resolution rate drops from 12.47% to 3.97%.",
       "evidence": "Figure 1b and Section 2.2. Only 91 of 251 patches classified as correct fixes (different from gold: 76, more comprehensive: 15).",
       "supported": "strong"
     },
     {
       "claim": "SWE-bench Lite and Verified also suffer from solution leakage (33.33% and 33.04% respectively).",
       "evidence": "Table 2, Section 2.3. Manual review of all issue reports in both variants.",
       "supported": "moderate"
     },
     {
       "claim": "On SWE-bench+, SWE-Agent+GPT-4 resolution rate drops to 0.55%.",
       "evidence": "Section 4, Table 3: only 3 correct fixes out of 548 instances.",
       "supported": "moderate"
     },
     {
       "claim": "AutoCodeRover+GPT-4o achieves the highest resolution rate on SWE-bench+ at 3.83% but drops from 18.83% on original leaderboard.",
       "evidence": "Section 4, Table 3: 21 correct patches out of 548 instances.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "SWE-bench has critical quality problems: 32.67% of successfully resolved issues contain leaked solutions in issue descriptions, and 31.08% pass despite incorrect/incomplete patches due to weak test cases. After filtering these problems, SWE-Agent+GPT-4's resolution rate drops from 12.47% to 3.97%. On the new SWE-bench+ dataset (post-training-cutoff issues with no solution leakage), resolution rates drop further to 0.55% for SWE-Agent+GPT-4, raising serious questions about current LLM coding benchmark validity.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section despite making strong claims about benchmark quality. Potential biases in manual classification (three annotators, no inter-rater reliability reported beyond 'disagreements resolved through discussion') are not discussed."
     },
     {
       "flag": "Confounded comparison",
       "detail": "SWE-bench+ differs from SWE-bench in multiple ways simultaneously: different time period, different repositories (Django excluded), different issue difficulty distribution. The performance drop cannot be attributed solely to removing leakage/weak tests vs. inherently harder recent issues."
     },
     {
       "flag": "No statistical testing",
       "detail": "Claims of 'significant' drops in resolution rates are made without any statistical significance tests. With small absolute numbers (e.g., 3 correct fixes out of 548), sampling variance could be substantial."
     },
     {
       "flag": "Single-run results",
       "detail": "All model evaluations appear to be single runs with no assessment of variance. LLM outputs are stochastic and results could differ across runs."
     },
     {
       "flag": "Inter-rater reliability not quantified",
       "detail": "Three authors classified 251 patches into patterns but no inter-rater agreement metric (Cohen's kappa, Fleiss' kappa) is reported. Only 'disagreements were resolved through discussion.'"
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "The original SWE-bench benchmark that this paper critiques and extends."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Primary evaluated tool; top SWE-bench performer at time of study."
     },
     {
       "title": "AutoCodeRover: Autonomous program improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "arxiv_id": "2404.05427",
       "relevance": "Evaluated as one of the top open-source SWE-bench models."
     },
     {
       "title": "Agentless: Demystifying LLM-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Alternative approach to LLM-based issue resolution evaluated on SWE-bench."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark for code generation; foundational code evaluation work."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2024,
       "relevance": "EvalPlus framework showing overestimation of LLM performance due to insufficient test cases — directly relevant to benchmark quality."
     },
     {
       "title": "CODER: Issue resolving with multi-agent and task graphs",
       "authors": ["Dong Chen"],
       "year": 2024,
       "arxiv_id": "2406.01304",
       "relevance": "Multi-agent approach to SWE-bench issue resolution."
     },
     {
       "title": "Diversity empowers intelligence: Integrating expertise of software engineering agents",
       "authors": ["Kexun Zhang"],
       "year": 2024,
       "arxiv_id": "2408.07060",
       "relevance": "Multi-agent framework for code generation showing impact of data diversity on LLM performance."
     },
     {
       "title": "SWE-bench-java: A GitHub issue resolving benchmark for Java",
       "authors": ["Daoguang Zan"],
       "year": 2024,
       "arxiv_id": "2408.14354",
       "relevance": "Extension of SWE-bench to Java; addresses language diversity in benchmark evaluation."
     },
     {
       "title": "MAGIS: LLM-based multi-agent framework for GitHub issue resolution",
       "authors": ["Wei Tao"],
       "year": 2024,
       "arxiv_id": "2403.17927",
       "relevance": "Multi-agent approach evaluated on SWE-bench."
     }
   ]
 }