ai-research-survey

scan-v4.json (21003B)

---

 {
   "scan_version": 4,
   "paper_type": "survey",
   "paper": {
     "title": "Dissecting the SWE-Bench Leaderboards: Profiling Submitters and Architectures of LLM- and Agent-Based Repair Systems",
     "authors": [
       "Matias Martinez",
       "Xavier Franch"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2506.17208",
     "doi": "10.1145/nnnnnnn.nnnnnnn"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about dominance of proprietary LLMs (especially Claude 3.5), presence of both agentic/non-agentic designs, and diverse contributor base are all supported by detailed results in Section 3.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is descriptive/observational and does not make causal claims. Statements like 'progress has been driven by advances in AI' are framing, not testable causal claims.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds its scope to SWE-Bench Lite and Verified leaderboards only, explains why other leaderboards (Full, Multimodal) were excluded (Section 2.1), and the threats to validity (Section 5) notes findings may not apply to other benchmarks.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses alternative explanations including patch overfitting inflating scores, data contamination, benchmark saturation, and that performance differences may reflect LLM capability rather than architectural choices.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly discusses that % Resolved (the proxy) may not reflect patch correctness (the outcome), citing Wang et al.'s finding that resolution rates are overstated by 6.2 percentage points due to overfitting patches.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Threats to Validity) provides a dedicated multi-page discussion covering external, internal, construct, and conclusion validity.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific threats: risk of missing documents for some approaches, challenges classifying submissions with limited descriptions, exclusion of monetary cost analysis due to changing token prices, and ambiguity in LLM usage information from informal sources.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it only studies SWE-Bench Lite and Verified (not Full or Multimodal), explains why, and notes 'we do not claim that our findings can be applied to' other benchmarks (Section 5, External Validity).",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors are affiliated with Universitat Politècnica de Catalunya, Spain. Neither appears to have conflicts with the evaluated systems.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key architectural dimensions are formally defined in Section 2.2: 'workflow authoring' (human vs. emergent), 'control flow autonomy' (emergent/scaffolded/fixed), and 'number of agents.' Submitter categories and product availability codes are also defined with explicit criteria.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper states explicitly that it presents 'the first comprehensive characterization of all submitters and solutions published on the SWE-Bench leaderboards,' guided by three clearly stated research questions (RQ1–RQ3).",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 contains substantive related work covering prior SWE-Bench empirical studies, approaches not in leaderboards, and APR on other benchmarks. The paper situates its contribution relative to patch quality studies (Wang et al., Aleithan et al.) and trajectory analyses (Ceka et al., Bouzenia et al.).",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": false,
           "justification": "Primary corpus is the leaderboard (anchored at a specific commit SHA), but secondary information gathering relies on Google searches, LinkedIn, and blog posts whose results would vary over time and are not archived — not fully reproducible.",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.1 explicitly excludes SWE-Bench Full (all solutions also in Lite/Verified) and Multimodal (restricted to language-based agents), with stated reasons. All entries in Lite and Verified up to July 17 are included.",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": false,
           "justification": "The paper uses content analysis methodology (deductive/inductive coding) but does not follow PRISMA or any other structured review protocol — no flow diagram, no formal protocol registration.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": true,
           "justification": "The Google query format is specified: '\"<Name_Entry> + SWE-Bench\"' with description of how entry names are constructed (e.g., 'GRU' from 'Gru(2024-12-08)').",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": true,
           "justification": "Information sources are explicitly listed: SWE-Bench leaderboard (with commit SHA), submission README.md/metadata.yaml files, Google Scholar, arXiv, LinkedIn, company websites, and blog posts.",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 shows documentation type distribution across entries; the paper reports 79 Lite entries, 99 Verified entries, 80 unique approaches, and explicitly notes 3 entries per leaderboard with no associated artifact.",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": true,
           "justification": "The choice of SWE-Bench Lite and Verified is justified by the benchmark's 'substantial impact on both academia and industry' (780+ citations), and exclusions (Full, Multimodal) are explicitly reasoned in Section 2.1.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.3 explicitly presents the single-agent vs. multi-agent debate with opposing views from Cognition, Anthropic, OpenHands, nFactorial, and Warp, concluding 'there is no one-size-fits-all solution.'",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": false,
           "justification": "The paper characterizes documentation types (Table 1: scientific articles vs blog posts vs README) but applies no formal quality rubric to assess the rigor of the approaches or their documentation.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss selection bias in which approaches are submitted to leaderboards (e.g., only high-performing systems are likely to be submitted), nor whether negative results are systematically absent.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": true,
           "justification": "Kruskal-Wallis tests with Dunn's post-hoc comparisons are applied to % Resolved by submitter type and architecture; median and max statistics are tabulated across multiple dimensions in Tables 2, 3, 4, 5, and 6.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": true,
           "justification": "The paper's summary conclusions (e.g., proprietary LLMs lead, no single architecture dominates, open-source catching up) are each tied to specific tables and statistical tests rather than to author opinion alone.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Industry submitters dominate SWE-Bench leaderboards, accounting for 58% of distinct submitters and increasing to 66% of Verified entries.",
       "evidence": "Table 2 and Figures 2/4 with entry counts by submitter type across both leaderboards.",
       "supported": "strong"
     },
     {
       "claim": "Proprietary LLMs, especially Claude 3.5 Sonnet and more recently Claude 4, have consistently achieved the highest % Resolved scores on both leaderboards.",
       "evidence": "Table 5 showing LLM combinations sorted by % Resolved; temporal Figure 9 showing Claude models dominating recent SOTA.",
       "supported": "strong"
     },
     {
       "claim": "No single architecture (single agent, multi-agent, fixed workflow) consistently achieves state-of-the-art performance.",
       "evidence": "Table 6 showing overlapping performance ranges across G1–G7; Kruskal-Wallis results showing no significant difference in Lite (p=0.0579).",
       "supported": "strong"
     },
     {
       "claim": "Open-source solutions are achieving competitive performance, with several reaching SOTA on both leaderboards in 2025.",
       "evidence": "Table 4 and Figure 7 showing open-source entries holding top rank intermittently; ExpeRepair (60.33%) and Moatless (70.8%) as examples.",
       "supported": "moderate"
     },
     {
       "claim": "Current % Resolved metrics overstate true repair performance by ~6.2 percentage points due to patch overfitting.",
       "evidence": "Cited from Wang et al. (2025), not the authors' own analysis — used to motivate discussion in Section 4.1.",
       "supported": "moderate"
     },
     {
       "claim": "SWE-Bench Verified may be approaching saturation, with July 2025 SOTA at 75.2% and all systems above 70% relying on Claude 4.",
       "evidence": "Figure 9b and Section 4.5 discussion; historical progression documented in Table 5.",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuned open-source models combined with closed-source LLMs can achieve near-SOTA performance (e.g., AgentScope at 63.4%).",
       "evidence": "Table 5 entry for AgentScope (Claude 3.5 Sonnet + Qwen2.5) and discussion in Section 3.1.5.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "observational",
     "qualitative",
     "meta-analysis"
   ],
   "key_findings": "Industry actors — especially small companies — dominate SWE-Bench submissions and have driven most SOTA improvements, particularly on SWE-Bench Verified. Proprietary LLMs (Claude 3.5/4 family) are used in the highest-performing entries, though fully open-source approaches are increasingly competitive. No single architectural paradigm (fixed workflow, scaffolded, emergent; single vs. multi-agent) consistently wins — top performance is achieved across diverse designs, suggesting that LLM quality is a stronger determinant than architecture. The benchmark may be approaching saturation at ~75%, and the paper highlights a systematic gap in correctness validation: % Resolved overstates true repair rates due to patch overfitting that is inconsistently addressed across submitters, especially outside the SE research community.",
   "red_flags": [
     {
       "flag": "No inter-rater reliability metric",
       "detail": "Architecture classifications relied on subjective interpretation of incomplete documentation. The paper mentions 'cross-validation among authors' but reports no Cohen's kappa or other reliability coefficient."
     },
     {
       "flag": "16 architecturally unclassifiable entries (G8)",
       "detail": "20% of Verified entries lack sufficient architectural documentation for classification. Findings about architecture-performance relationships may be biased toward better-documented (often academic) submissions."
     },
     {
       "flag": "Google search results not archived",
       "detail": "Secondary information gathering via Google, LinkedIn, and blog posts is not reproducible — results would differ if the study were repeated as pages disappear or change."
     },
     {
       "flag": "No funding disclosure",
       "detail": "Neither author acknowledgment nor funding statement appears anywhere in the paper, preventing assessment of potential conflicts with the commercial systems evaluated."
     },
     {
       "flag": "Self-reported/inferred architecture data",
       "detail": "Architectural classifications depend on what submitters chose to disclose (blog posts, READMEs), creating a systematic bias toward more transparent (typically academic or open-source) systems."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "The primary benchmark being analyzed; provides the leaderboard infrastructure and defines the % Resolved metric."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "relevance": "One of the most widely analyzed submissions; used as a baseline and extended by many other approaches."
     },
     {
       "title": "Agentless: Demystifying LLM-Based Software Engineering Agents",
       "relevance": "Another widely-extended submission representing the non-agentic workflow paradigm; many leaderboard entries build on it."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "relevance": "Key academic submission analyzed across both leaderboards; represents early SOTA in scaffolded multi-agent design."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "relevance": "Key open-source platform with multiple submissions; represents single-agent emergent design paradigm."
     },
     {
       "title": "Large Language Model-Based Agents for Software Engineering: A Survey",
       "relevance": "Provides the end-to-end software maintenance pipeline taxonomy used in RQ3 analysis."
     },
     {
       "title": "Are 'Solved Issues' in SWE-Bench Really Solved Correctly? An Empirical Study",
       "relevance": "Key related work on patch overfitting in SWE-Bench; provides the 6.2pp overstatement estimate cited in the discussion."
     },
     {
       "title": "Introducing SWE-bench Verified",
       "relevance": "Describes the construction of SWE-Bench Verified leaderboard and its curation criteria."
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "relevance": "Provides taxonomy of multi-agent failure modes, directly relevant to the single vs. multi-agent architectural debate."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Provides a taxonomy of repair architectures but no directly usable tool or technique practitioners can apply at work."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that no single architecture dominates is mildly surprising given hype around agentic systems, but most other findings confirm conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk angle is discussed."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Directly examines SWE-Bench limitations including potential saturation, patch overfitting inflating scores by ~6pp, and questions whether industry submitters account for correctness."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Pure observational study with no code, demo, or interactive artifact to try."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Centers on the widely-discussed SWE-Bench benchmark and references major companies (Anthropic, Google, Amazon, OpenAI) and products (Claude, Copilot-adjacent tools)."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "44489690",
         "title": "Mercury: Ultra-fast language models based on diffusion",
         "points": 576,
         "comments": 242,
         "url": "https://news.ycombinator.com/item?id=44489690",
         "created_at": "2025-07-07T12:31:08Z"
       },
       {
         "hn_id": "44412427",
         "title": "Mercury: Ultra-Fast Language Models Based on Diffusion",
         "points": 10,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=44412427",
         "created_at": "2025-06-29T12:05:48Z"
       },
       {
         "hn_id": "44358841",
         "title": "Machine Mental Imagery: Empower Multimodal Reasoning with Latent Visual Tokens",
         "points": 7,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44358841",
         "created_at": "2025-06-23T18:52:55Z"
       },
       {
         "hn_id": "44101770",
         "title": "Effective Reinforcement Learning for Reasoning in Language Models",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44101770",
         "created_at": "2025-05-26T21:17:20Z"
       },
       {
         "hn_id": "44314613",
         "title": "Wanting to Be Understood Explains the Meta-Problem of Consciousness",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44314613",
         "created_at": "2025-06-19T01:16:41Z"
       },
       {
         "hn_id": "44304578",
         "title": "Serving Large Language Models on Huawei CloudMatrix384",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44304578",
         "created_at": "2025-06-17T22:18:43Z"
       },
       {
         "hn_id": "44009979",
         "title": "A Search for Planet Nine with IRAS and Akari Data",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44009979",
         "created_at": "2025-05-16T21:35:58Z"
       },
       {
         "hn_id": "46445614",
         "title": "Mechanical non-reciprocity programmed by shear jamming in soft composite solids",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46445614",
         "created_at": "2025-12-31T16:32:15Z"
       },
       {
         "hn_id": "44047429",
         "title": "Model Merging in Pre-Training of Large Language Models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44047429",
         "created_at": "2025-05-21T01:12:29Z"
       },
       {
         "hn_id": "42816449",
         "title": "Dissecting the NVIDIA Hopper Architecture through Microbenchmarking",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42816449",
         "created_at": "2025-01-24T20:02:41Z"
       }
     ],
     "top_points": 576,
     "total_points": 612,
     "total_comments": 244
   }
 }