ai-research-survey

scan.json (21925B)

---

 {
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "paper": {
     "title": "What's in a Benchmark? The Case of SWE-Bench in Automated Program Repair",
     "authors": ["Matias Martinez", "Xavier Franch"],
     "year": 2026,
     "venue": "ICSE-SEIP 2026",
     "arxiv_id": "2602.04449",
     "doi": "10.1145/3786583.3786904"
   },
   "methodology_tags": ["observational", "meta-analysis"],
   "key_findings": "Analysis of 212 entries across SWE-Bench Lite and Verified leaderboards shows industry dominates submissions (58-79% depending on how collaborations are counted), with small companies and large publicly traded firms leading. Proprietary LLMs, especially the Claude family, achieve the highest precision, with Claude 4 Sonnet powering all entries above 70% on Verified. Open-source solutions remain competitive but trail in median precision. The paper identifies critical issues including patch overfitting, data contamination, limited real-world representation (Python-only), and a transparency-participation trade-off in metadata requirements.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Appendix data is hosted at https://github.com/UPCBarcelonaTech/DissectingSWEBench (reference [3])."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The complete list of approaches and submissions is provided in the appendix repository [3], and the paper references metadata collected from leaderboard entries."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification provided. The study is primarily manual analysis, but any analysis scripts would benefit from environment documentation."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology section describes the process but does not provide a reproducible script or checklist for replicating the data collection."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. Tables show medians and maxima but no uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Kruskal-Wallis test used for comparing precision across submitter types (H=19.89, p=0.0469 for Lite; H=38.0953, p=0.0001 for Verified). Dunn's post-hoc test applied for Verified with specific p-values reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only H statistics and p-values are reported from statistical tests. No formal effect size measures (e.g., eta-squared, rank-biserial correlation) are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample is the full population of leaderboard entries (79 Lite, 133 Verified), but there is no discussion of whether this is sufficient for the statistical tests performed, nor any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only medians and maxima are reported. No standard deviations, IQR, or other spread measures are provided for the precision distributions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "This is described as 'the first comprehensive study' of SWE-Bench leaderboards. No comparison with prior survey results or alternative analysis approaches is provided."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included, so contemporaneity cannot be assessed."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple dimensions analyzed: % Resolved, number of entries, submitter types, product availability modes, open-source share, LLM usage patterns."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of system outputs is not relevant to this leaderboard analysis study."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to this observational study; there is no prediction task requiring train/test separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns provided: by submitter type (Table 1), product availability and form (Table 2), open vs closed source (Table 3), and LLM combinations (Table 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses entries that could not be fully characterized ('Unknown' categories), country analysis that was discarded due to unreliable data, and limitations of the coding approach."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Reports that open-source LLMs yield lower precision, that academic median precision is lower than industry, and that some planned analyses (country-level) had to be abandoned."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about industry dominance, Claude family dominance, and open-source competitiveness are all supported by Tables 1-4 and the detailed analysis in Section 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper avoids strong causal language, using hedged phrasing like 'This difference may be attributed to the construction of Verified' and 'may be influenced by the early submissions.' The Kruskal-Wallis tests support comparative claims appropriately."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (External Validity): 'Other benchmarks may be equally representative of the broader issue-fixing landscape, but we do not claim that our findings can be applied to them.' Scope explicitly limited to two SWE-Bench leaderboards."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple alternative explanations discussed: early academic submissions influencing lower median precision, construction of Verified filtering affecting precision differences, selective reporting inflating apparent saturation, cost barriers explaining open-source underperformance."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses that % Resolved (test-passing) is a proxy for correctness, with Section 4 'Patch Overfitting on SWE-Bench' explaining that overfitting patches pass tests but remain incorrect, and 'Limited Real-World Representation' noting SWE-bench may not generalize."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No AI models are used in the study methodology itself."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in this study."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No models or hyperparameters are used in this observational study."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the data collection pipeline in detail: visiting leaderboard pages, following links, Google searches with specific query format, LinkedIn inspection, and metadata extraction from README.md and metadata.yaml files."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Threats to Validity' provides substantive discussion across external, internal, and construct validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: risk of missing artifacts for some entries, country analysis discarded due to unreliable data, LLM parameter counts not analyzed, cost comparisons excluded due to declining token prices making cross-temporal comparison misleading."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly states: excluded Full and Multimodal leaderboards with reasons given, does not claim generalization to other benchmarks, did not analyze LLM parameter counts or monetary costs."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Raw data provided in appendix repository at https://github.com/UPCBarcelonaTech/DissectingSWEBench (reference [3]), including complete lists of approaches and submissions."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the full data collection procedure including visiting leaderboard pages, following links, Google searches, LinkedIn profiles, and metadata files."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data source is public leaderboard entries."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2-2.3 documents the pipeline from visiting leaderboards to content analysis coding. Steps (a)-(f) are enumerated for each entry's data collection. Section 2.3 describes the coding methodology including deductive and inductive approaches."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section discloses: 'Ramon y Cajal' Fellowship (RYC2021-031523-I), grant PID2024-156019OB-I00 funded by MICIU/AEI and ERDF."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors affiliated with Universitat Politècnica de Catalunya, clearly stated. No conflict with SWE-bench or any evaluated product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funded by Spanish government research grants with no financial stake in SWE-bench outcomes or any of the evaluated products/companies."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This study does not evaluate a pre-trained model on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This study does not evaluate a pre-trained model on any benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This study does not evaluate a pre-trained model on any benchmark. However, the paper does discuss contamination as a meta-concern for SWE-bench submissions in Section 4."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/observational study with no computational method whose cost needs reporting."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/observational study; no significant compute was required."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "No PRISMA flow diagram or formal review protocol is referenced. The methodology uses content analysis with inductive/deductive coding (Section 2.3) but does not follow an established systematic review protocol."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The study catalogs and classifies leaderboard entries but does not assess the methodological quality of the underlying submissions or their associated papers. All entries are treated equally regardless of rigor."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 'Saturation and the Cherry-Picking of Submissions' explicitly discusses selective reporting: 'submissions may test several LLMs but only the best outcome is submitted, hiding result variability.' Also discusses how leaderboard participation is biased toward well-resourced actors."
       }
     }
   },
   "claims": [
     {
       "claim": "The majority of SWE-Bench submissions originate from industry, representing 58% of submitters (79% including industry-academia collaborations).",
       "evidence": "Table 1 shows 45 distinct industry submitters across both leaderboards, with 106 total industry entries. Section 3.2 provides detailed breakdown.",
       "supported": "strong"
     },
     {
       "claim": "Proprietary LLMs, especially Claude 4 Sonnet, achieve the highest precision on both leaderboards, with all entries exceeding 70% on Verified using Claude 4 models.",
       "evidence": "Table 4 shows Claude 4 Sonnet achieving max 76.8% on Verified and 60.33% on Lite. All top-10 entries on Verified use Claude 4 family models.",
       "supported": "strong"
     },
     {
       "claim": "Small companies have been consistently driving improvements in state-of-the-art precision on both leaderboards.",
       "evidence": "Figure 2 and Section 3.2 show small companies (e.g., Isoform, Refact.ai) repeatedly pushing top results. Table 1 shows 24 distinct small company submitters with 52 entries.",
       "supported": "strong"
     },
     {
       "claim": "There is a statistically significant difference in precision between Single Academia and Small Companies on Lite (H=19.89, p=0.0469).",
       "evidence": "Kruskal-Wallis test reported in Section 3.2. However, the authors note this may be influenced by early low-precision academic submissions.",
       "supported": "moderate"
     },
     {
       "claim": "Open-source solutions remain competitive with closed-source, with the top performer on Lite being open source and several open-source approaches achieving near-SOTA on Verified.",
       "evidence": "Table 3 shows open-source max of 60.33% on Lite (highest overall) and 75.2% on Verified. However, median precision favors closed-source.",
       "supported": "strong"
     },
     {
       "claim": "Activity on SWE-Bench Lite has significantly dropped in 2025, with the most recent entry dated June 27, 2025.",
       "evidence": "Figure 1a and Section 3.1 document this trend, showing submissions concentrating on Verified.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No inter-rater reliability for coding",
       "detail": "The content analysis coding (submitter type, product category, availability) was performed but the paper does not mention whether multiple coders were used or inter-rater reliability was assessed. For an inductive coding approach, this is a significant omission."
     },
     {
       "flag": "Snapshot-dependent results",
       "detail": "Data collected as of September 18, 2025. Leaderboard entries and their metadata can change, be updated, or be removed. The findings are inherently time-bound but presented as general characterizations."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "The foundational SWE-bench benchmark paper, central to the study's scope."
     },
     {
       "title": "Agentless: Demystifying llm-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Key open-source APR approach evaluated on SWE-bench, representing the non-agentic alternative."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "relevance": "Influential agentic APR system with multiple leaderboard entries using different LLMs."
     },
     {
       "title": "OpenHands: An open platform for ai software developers as generalist agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "relevance": "Open-source agentic development platform evaluated on SWE-bench."
     },
     {
       "title": "Alibaba lingmaagent: Improving automated issue resolution via comprehensive repository exploration",
       "authors": ["Yingwei Ma", "Qingping Yang", "Rongyu Cao", "Binhua Li", "Fei Huang", "Yongbin Li"],
       "year": 2025,
       "relevance": "Industry-backed open-source agent achieving competitive SWE-bench results."
     },
     {
       "title": "The swe-bench illusion: When state-of-the-art llms remember instead of reason",
       "authors": ["Shanchao Liang", "Spandan Garg", "Roshanak Zilouchian Moghaddam"],
       "year": 2025,
       "relevance": "Investigates data contamination in SWE-bench, finding models may memorize rather than reason."
     },
     {
       "title": "Are 'solved issues' in swe-bench really solved correctly? an empirical study",
       "authors": ["You Wang", "Michael Pradel", "Zhongxin Liu"],
       "year": 2025,
       "relevance": "Studies patch overfitting on SWE-bench, finding 7.8% of plausible patches are incorrect."
     },
     {
       "title": "Revisiting SWE-Bench: On the Importance of Data Quality for LLM-Based Code Models",
       "authors": ["Reem Aleithan"],
       "year": 2025,
       "relevance": "Manual analysis of SWE-bench patches finding 32.67% had complete solutions leaked in issue descriptions."
     },
     {
       "title": "Repairagent: An autonomous, llm-based agent for program repair",
       "authors": ["Islem Bouzenia", "Premkumar Devanbu", "Michael Pradel"],
       "year": 2024,
       "relevance": "LLM-based autonomous program repair agent evaluated on SWE-bench."
     },
     {
       "title": "Trae agent: An llm-based agent for software engineering with test-time scaling",
       "authors": ["Trae Research Team"],
       "year": 2025,
       "relevance": "ByteDance's agent achieving 75.2% on SWE-bench Verified using multiple Claude models."
     },
     {
       "title": "RepairBench: Leaderboard of frontier models for program repair",
       "authors": ["André Silva", "Martin Monperrus"],
       "year": 2025,
       "relevance": "Alternative benchmark/leaderboard for program repair evaluation."
     },
     {
       "title": "Swe-bench goes live!",
       "authors": ["Linghao Zhang", "Shilin He", "Chaoyun Zhang"],
       "year": 2025,
       "relevance": "SWE-bench-Live: automated pipeline for continuously updating the benchmark with recent data to address contamination."
     }
   ]
 }