ai-research-survey

scan.json (24006B)

---

 {
   "paper": {
     "title": "Dissecting the SWE-Bench Leaderboards: Profiling Submitters and Architectures of LLM- and Agent-Based Repair Systems",
     "authors": [
       "Matias Martinez",
       "Xavier Franch"
     ],
     "year": 2026,
     "venue": "arXiv (Manuscript submitted to ACM)",
     "arxiv_id": "2506.17208"
   },
   "scan_version": 3,
   "active_modules": [
     "survey_methodology"
   ],
   "methodology_tags": [
     "meta-analysis",
     "observational",
     "qualitative"
   ],
   "key_findings": "The study analyzes 178 entries (80 unique approaches) across SWE-Bench Lite and Verified leaderboards. Industry submitters dominate (58% of distinct submitters), with small companies being the largest category. Proprietary LLMs, especially Claude 3.5/4 Sonnet, consistently achieve highest performance. No single architecture (agentic vs non-agentic, single vs multi-agent) consistently outperforms others, though submissions without agents (G1) tend to show lower precision.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The appendix data is available at https://github.com/UPCBarcelonaTech/DissectingSWEBench (referenced as [5])."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The complete list of approaches and submissions is provided in the appendix [5] at a GitHub repository, and the study is based on publicly available SWE-Bench leaderboard data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications or dependency requirements are mentioned for reproducing the analysis."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided for the data collection and coding process."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports medians and maximums but no confidence intervals or error bars on its aggregate statistics."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Kruskal-Wallis tests and Dunn's post-hoc tests to compare precision across submitter types (Section 3.1.2) and architecture groups (Section 3.2.9), reporting H-statistics and p-values."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute percentage differences in precision across groups (e.g., median 46.9% vs 31.5% for Verified vs Lite) providing baseline context for interpreting differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The study analyzes all entries on the leaderboards (79 + 99 = 178), which is a census not a sample, but no discussion of statistical power for the subgroup comparisons is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only medians and maximums are reported for % Resolved across groups. No standard deviations, IQR, or other spread measures are provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "This is the first study of its kind on SWE-Bench leaderboards, and no comparison against prior surveys or analyses is made."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No prior comprehensive study of SWE-Bench leaderboards exists to serve as a baseline."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/observational study with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The study analyzes submissions across multiple dimensions: % Resolved, submitter type distribution, product availability, open-source status, LLM usage, and architectural classification."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of system outputs is not relevant to this survey of leaderboard submissions."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey study, not an experiment with train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns are provided by submitter type, product availability, open-source status, LLM family, architecture group (G1-G8), and per-leaderboard (Lite vs Verified)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses patch overfitting problems, multi-agent failure modes (citing Cemri et al.'s 14 failure modes), benchmark limitations, and data contamination concerns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that no single architecture consistently achieves SOTA, that open-source-only LLM submissions achieve lower performance, and that some multi-agent approaches were abandoned in favor of single-agent (nFactorial, Warp examples in Section 4.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about dominance of proprietary LLMs (especially Claude 3.5), presence of both agentic/non-agentic designs, and diverse contributor base are all supported by detailed results in Section 3."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is descriptive/observational and does not make causal claims. Statements like 'progress has been driven by advances in AI' are framing, not testable causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds its scope to SWE-Bench Lite and Verified leaderboards only, explains why other leaderboards (Full, Multimodal) were excluded (Section 2.1), and the threats to validity (Section 5) notes findings may not apply to other benchmarks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses alternative explanations including patch overfitting inflating scores, data contamination, benchmark saturation, and that performance differences may reflect LLM capability rather than architectural choices."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly discusses that % Resolved (the proxy) may not reflect patch correctness (the outcome), citing Wang et al.'s finding that resolution rates are overstated by 6.2 percentage points due to overfitting patches."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use LLMs in its methodology. It analyzes other submissions' LLM usage."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. It is a manual survey study."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No LLMs or models are run as part of this study's methodology."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1.1 describes the detailed data collection procedure: visiting leaderboard pages, following links, conducting Google searches with specific query templates, inspecting README/metadata files, and checking LinkedIn. Section 2.1.2 describes the coding schema development using inductive/deductive coding."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Threats to Validity) provides a dedicated multi-page discussion covering external, internal, construct, and conclusion validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific threats: risk of missing documents for some approaches, challenges classifying submissions with limited descriptions, exclusion of monetary cost analysis due to changing token prices, and ambiguity in LLM usage information from informal sources."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it only studies SWE-Bench Lite and Verified (not Full or Multimodal), explains why, and notes 'we do not claim that our findings can be applied to' other benchmarks (Section 5, External Validity)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The complete list of approaches and submissions is provided in the appendix [5] at a GitHub repository, and the underlying SWE-Bench leaderboard data is publicly accessible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1.1 provides a detailed step-by-step data collection procedure including visiting leaderboard pages, following links, Google searches, inspecting metadata files, and LinkedIn profiles."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyzes publicly available leaderboard submissions."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the full pipeline: leaderboard scraping → artifact collection → content analysis with coding schema → classification across multiple dimensions. Table 1 shows distribution of artifact types found."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors are affiliated with Universitat Politècnica de Catalunya, Spain. Neither appears to have conflicts with the evaluated systems."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model on any benchmark. It is a survey of leaderboard submissions."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model on any benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model on any benchmark. However, it does discuss contamination as a limitation of SWE-Bench itself (Section 4.4)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational method to cost."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not follow PRISMA or any named structured review protocol. While the data collection is systematic (Section 2.1.1), there is no PRISMA flow diagram, no protocol registration, and the search strategy is specific to leaderboard scraping rather than a reproducible literature search."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the quality of the source papers or blog posts used to characterize submissions. All sources are treated equally regardless of whether they are peer-reviewed papers, blog posts, or README files. The paper discusses patch overfitting concerns (Section 4.1) but does not systematically assess source quality."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not discuss publication bias in the leaderboard context — e.g., whether submissions with poor results are less likely to be submitted, creating a survivorship bias in the analyzed entries."
       }
     }
   },
   "claims": [
     {
       "claim": "The majority of submissions come from industry, particularly small companies and large publicly traded corporations, with industry accounting for 58% of distinct submitters.",
       "evidence": "Table 2 and Figure 4 show 41 out of 71 distinct submitters (58%) are from industry. Section 3.1.2 provides detailed breakdown.",
       "supported": "strong"
     },
     {
       "claim": "Proprietary LLMs, especially Claude 3.5/4 Sonnet, consistently achieve the highest precision on both leaderboards.",
       "evidence": "Table 5 shows Claude 4 Sonnet achieving 60.33% max on Lite and Claude 4 variants dominating top entries on Verified (up to 75.2%). Section 3.1.5 details the LLM analysis.",
       "supported": "strong"
     },
     {
       "claim": "No single architecture consistently achieves state-of-the-art performance across both leaderboards.",
       "evidence": "Table 6 shows top performance achieved by G3 (75.2% on Verified), G4 (60% on Lite), and G6 (73.2% on Verified). Kruskal-Wallis test on Lite showed no significant difference (p=0.0579). Section 3.2.9.",
       "supported": "strong"
     },
     {
       "claim": "Closed-source solutions exhibit higher median % Resolved, but open-source tools are showing competitive and sometimes state-of-the-art performance.",
       "evidence": "Table 4 shows closed-source median of 35.67% vs 30.33% on Lite and 50% vs 44.2% on Verified, but top entries on both are open-source.",
       "supported": "strong"
     },
     {
       "claim": "Since the introduction of SWE-Bench Verified, companies (especially small ones) have been driving progress toward state-of-the-art performance.",
       "evidence": "Figure 5b and Section 3.1.2 show small companies consistently pushing top results, with Refact.ai (74.4%), Bytedance (75.2%) on Verified.",
       "supported": "strong"
     },
     {
       "claim": "SWE-Bench may be approaching saturation, with submissions achieving up to 75% precision.",
       "evidence": "Section 4.5 notes 20+ percentage point improvement over previous year, with all >70% submissions using Claude 4 models. Argument is based on trend extrapolation.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No quality assessment of sources",
       "detail": "The survey treats blog posts, LinkedIn posts, and README files as equivalent to peer-reviewed papers when characterizing approaches. No assessment of source reliability or completeness is performed."
     },
     {
       "flag": "Survivorship bias in leaderboard analysis",
       "detail": "Only submissions that were actually made to the leaderboard are analyzed. Approaches that performed poorly and were never submitted, or submissions that were withdrawn, are invisible. This could bias conclusions about what architectures work."
     },
     {
       "flag": "Classification subjectivity with limited inter-rater reliability data",
       "detail": "The architectural classification (G1-G8) relies on manual content analysis by two authors, but no inter-rater reliability metrics (Cohen's kappa, etc.) are reported. The paper states annotations were 'cross-validated among the authors' but provides no quantitative agreement measure."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": [
         "Carlos E. Jimenez",
         "John Yang",
         "Alexander Wettig",
         "Shunyu Yao",
         "Kexin Pei",
         "Ofir Press",
         "Karthik Narasimhan"
       ],
       "year": 2024,
       "relevance": "The benchmark being studied; foundational to the entire SWE-bench ecosystem analyzed in this paper."
     },
     {
       "title": "Agentless: Demystifying LLM-based Software Engineering Agents",
       "authors": [
         "Chunqiu Steven Xia",
         "Yinlin Deng",
         "Soren Dunn",
         "Lingming Zhang"
       ],
       "year": 2024,
       "relevance": "Pioneering non-agentic approach on SWE-bench that spawned multiple extensions and variants."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": [
         "John Yang",
         "Carlos E. Jimenez",
         "Alexander Wettig",
         "Kilian Lieret",
         "Shunyu Yao",
         "Karthik Narasimhan",
         "Ofir Press"
       ],
       "year": 2024,
       "relevance": "Foundational single-agent system for SWE-bench, demonstrating emergent workflow with ReAct."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": [
         "Xingyao Wang",
         "Boxuan Li"
       ],
       "year": 2024,
       "relevance": "Open platform for AI coding agents, advocates single-agent architecture, competitive SWE-bench results."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": [
         "Yuntong Zhang",
         "Haifeng Ruan",
         "Zhiyu Fan",
         "Abhik Roychoudhury"
       ],
       "year": 2024,
       "relevance": "Multi-agent scaffolded workflow for SWE-bench, later acquired by Sonar and extended to SpecRover."
     },
     {
       "title": "MASAI: Modular Architecture for Software-Engineering AI Agents",
       "authors": [
         "Daman Arora"
       ],
       "year": 2024,
       "relevance": "Multi-agent modular architecture with specialized sub-agents for different repair phases."
     },
     {
       "title": "Large Language Model-based Agents for Software Engineering: A Survey",
       "authors": [
         "Junwei Liu",
         "Kaixin Wang"
       ],
       "year": 2024,
       "relevance": "Provides the end-to-end software maintenance pipeline taxonomy used as the analytical framework for RQ3."
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": [
         "Mert Cemri"
       ],
       "year": 2025,
       "relevance": "Empirical study of multi-agent failure modes in SWE-bench systems, identifying 14 distinct failure types."
     },
     {
       "title": "Are 'Solved Issues' in SWE-Bench Really Solved Correctly? An Empirical Study",
       "authors": [
         "You Wang",
         "Michael Pradel",
         "Zhongxin Liu"
       ],
       "year": 2025,
       "relevance": "Found SWE-bench resolution rates are overstated by 6.2 percentage points due to overfitting patches."
     },
     {
       "title": "PatchPilot: A Stable and Cost-Efficient Agentic Patching Framework",
       "authors": [
         "Hongwei Li",
         "Yuheng Tang",
         "Shiqi Wang",
         "Wenbo Guo"
       ],
       "year": 2025,
       "relevance": "Multi-stage agentic patching framework with refinement component, achieving competitive SWE-bench results."
     },
     {
       "title": "SWE-RL: Advancing LLM Reasoning via Reinforcement Learning on Open Software Evolution",
       "authors": [
         "Yuxiang Wei"
       ],
       "year": 2025,
       "relevance": "Uses reinforcement learning to train LLMs for autonomous developer reasoning, evaluated on SWE-bench."
     },
     {
       "title": "The SWE-Bench Illusion: When State-of-the-Art LLMs Remember Instead of Reason",
       "authors": [
         "Shanchao Liang",
         "Spandan Garg",
         "Roshanak Zilouchian Moghaddam"
       ],
       "year": 2025,
       "relevance": "Investigates whether SWE-bench performance is driven by memorization rather than genuine coding capability."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Provides a taxonomy of repair architectures but no directly usable tool or technique practitioners can apply at work."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that no single architecture dominates is mildly surprising given hype around agentic systems, but most other findings confirm conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk angle is discussed."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Directly examines SWE-Bench limitations including potential saturation, patch overfitting inflating scores by ~6pp, and questions whether industry submitters account for correctness."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Pure observational study with no code, demo, or interactive artifact to try."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Centers on the widely-discussed SWE-Bench benchmark and references major companies (Anthropic, Google, Amazon, OpenAI) and products (Claude, Copilot-adjacent tools)."
     }
   }
 }