ai-research-survey

scan-v5.json (20598B)

---

 {
   "scan_version": 5,
   "paper_type": "survey",
   "paper": {
     "title": "Dissecting the SWE-Bench Leaderboards: Profiling Submitters and Architectures of LLM- and Agent-Based Repair Systems",
     "authors": [
       "Matias Martinez",
       "Xavier Franch"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2506.17208",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims—first comprehensive study, 80 unique approaches, Claude dominance, architectural diversity, contributor diversity—are substantiated by Tables 2–6 and the RQ results sections.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is descriptive and observational; it makes no formal causal claims, only correlational or descriptive observations about leaderboard submissions.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (External Validity) explicitly states the findings are bounded to SWE-Bench Lite and Verified and that 'we do not claim that our findings can be applied to them' (other benchmarks).",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 presents competing perspectives on single- vs. multi-agent architectures from Cognition, Anthropic, OpenHands, and nFactorial; Section 3.1.2 attributes early academic underperformance partly to temporal effects rather than capability.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly discusses that '% Resolved' conflates plausible and correct patches, citing Wang et al.'s finding of a 6.2pp average overstatement, and calls for additional validation beyond test suites.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Threats to Validity' contains four dedicated subsections: External, Internal, Construct, and Conclusion Validity.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are concrete: risk of missing submission documentation (Internal), exclusion of monetary cost analysis due to token-price normalization difficulty, G8 category for architecturally unclassifiable entries, and content analysis limitations (Construct).",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 explicitly excludes Full and Multimodal leaderboards with rationale; data cutoff is July 17th pinned to a specific GitHub commit hash.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned anywhere in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors are identified as affiliated with Universitat Politècnica de Catalunya, Barcelona, Spain, with contact emails provided.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funder cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests appears in the paper despite analyzing commercial tools from major AI companies.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 formally defines workflow authoring (human vs. emergent), control flow autonomy (emergent, scaffolded, fixed), and agent count categories; Section 2.1.2 defines submitter categories with explicit coding schemas.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it presents 'the first in-depth study of the SWE-Bench leaderboards' with three clearly stated research questions (RQ1–RQ3) covering submitter profiling, architecture, and pipeline phases.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 engages substantively with prior empirical studies of SWE-Bench patches (Meng et al., Wang et al., Aleithan et al., Ceka et al.) and distinguishes this leaderboard-level characterization from their patch-level analyses.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": true,
           "justification": "The leaderboard data source is pinned to a specific GitHub commit hash; Section 2.1.1 provides the Google supplementary query format ('<Name_Entry> + SWE-Bench') with a worked example.",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": true,
           "justification": "Inclusion: all entries on Lite and Verified as of July 17th. Exclusion: Full (all solutions are subsets of Lite/Verified) and Multimodal (language-based focus only), with explicit rationale for each.",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": false,
           "justification": "No PRISMA or equivalent structured review protocol is followed; the paper uses content analysis on leaderboard data rather than a systematic literature review protocol with formal screening stages.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.1.1 explicitly provides the Google query format: '<Name_Entry> + SWE-Bench' with example 'GRU' from entry 'Gru(2024-12-08)'.",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": true,
           "justification": "Sources explicitly listed: SWE-Bench leaderboard pages, SWE-Bench GitHub repository (experiments), Google search, LinkedIn, arXiv, and scientific publications.",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": false,
           "justification": "Table 1 shows artifact type distribution but there is no PRISMA-style flow diagram with counts at each screening stage (records identified → screened → excluded → included).",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.1 explains the choice of Lite and Verified (high impact, full coverage of other leaderboards), the July 17th cutoff, and the exclusion of non-language modalities.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.3 presents substantive disagreement between Cognition (anti-multi-agent), Anthropic (pro-multi-agent for their use case), OpenHands (pro-single-agent), and empirical evolution of nFactorial across four submissions.",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": false,
           "justification": "The paper categorizes submissions by documentation type (Table 1) and introduces G8 for unclassifiable entries but applies no formal quality rubric or risk-of-bias assessment to source papers or approaches.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss leaderboard submission bias (e.g., that only successful approaches are submitted, that negative results are never shared), which is a relevant concern for interpreting apparent progress trends.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": true,
           "justification": "Kruskal-Wallis tests with Dunn's post-hoc comparisons are applied to compare % Resolved across submitter types and architecture groups (Tables 2, 6); median and maximum precision reported for all categories.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": true,
           "justification": "Recommendations (semantic correctness validation, open-source framework value) directly follow from documented empirical findings: Wang et al.'s overfitting data and the SIMA/Augment Code open-source success cases.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Industry accounts for 58% of distinct submitters and 65% of entries in SWE-Bench Verified, with small companies the dominant subtype.",
       "evidence": "Table 2 and Figures 2–4 show 41/71 distinct submitters from industry; 65/99 Verified entries are from industry; 15–16 of those are small companies.",
       "supported": "strong"
     },
     {
       "claim": "Proprietary LLMs—especially Claude 3.5 and Claude 4 families—consistently achieve the highest precision on both leaderboards.",
       "evidence": "Table 5 shows Claude 3.5 Sonnet is the most-used model; Section 4.5 notes all systems exceeding 70% on Verified use Claude 4 models.",
       "supported": "strong"
     },
     {
       "claim": "No single architecture consistently achieves state-of-the-art performance across both leaderboards.",
       "evidence": "Table 6 and Kruskal-Wallis tests: no statistically significant architecture differences in Lite (p=0.0579); G3 tops Verified max but G6 (31 entries) is largest and competitive.",
       "supported": "strong"
     },
     {
       "claim": "Open-source solutions are approaching competitive performance with closed-source, with several reaching state-of-the-art in 2025.",
       "evidence": "Table 4 shows top-ranked entries in both leaderboards are open-source; Figure 7 shows convergence of open- and closed-source performance in 2025.",
       "supported": "strong"
     },
     {
       "claim": "SWE-Bench may be approaching saturation, with 75.2% precision reached in July 2025 versus ~50% one year earlier.",
       "evidence": "Figure 1b documents the progression; Section 4.5 draws HumanEval saturation analogy but notes this is a projection, not established fact.",
       "supported": "moderate"
     },
     {
       "claim": "Current SWE-Bench evaluation overstates resolution rates by ~6.2 percentage points due to patch overfitting.",
       "evidence": "Section 4.1 cites Wang et al. [75] who ran PatchDiff on three systems; this finding is not independently verified in the present paper.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "observational",
     "meta-analysis",
     "qualitative"
   ],
   "key_findings": "This first comprehensive characterization of SWE-Bench Lite (79 entries) and Verified (99 entries) leaderboards finds that industry—especially small companies—dominates submissions (65% of Verified entries), while proprietary LLMs (Claude 3.5/4) consistently achieve highest precision. No single architecture reliably outperforms: human-authored multi-agent fixed workflows (G3) and scaffolded single-agent (G4) approaches top SWE-Bench Lite, while emergent single-agent systems (G6) are the most numerous and competitive on Verified. Open-source approaches became increasingly competitive throughout 2025. The benchmark shows saturation signals at 75% precision, and its test-passing metric likely overstates true resolution rates due to patch overfitting.",
   "red_flags": [
     {
       "flag": "No funding disclosure",
       "detail": "No funding source is mentioned anywhere in the paper, making it impossible to assess potential financial conflicts of interest."
     },
     {
       "flag": "No competing interests statement",
       "detail": "Neither author declares financial interests despite the paper profiling commercial tools from Anthropic, Google, Amazon, IBM, and others."
     },
     {
       "flag": "Non-reproducible supplementary search",
       "detail": "Google search results and LinkedIn browsing used to supplement leaderboard data cannot be exactly reproduced; results vary by user, date, and locale."
     },
     {
       "flag": "Large unclassifiable subset",
       "detail": "13 Lite entries and 16 Verified entries (G8) cannot be architecturally classified due to insufficient public documentation, limiting the scope of architectural conclusions."
     },
     {
       "flag": "No PRISMA or formal screening flow",
       "detail": "Despite systematically reviewing a corpus of submissions, the paper omits a PRISMA-style screening diagram with counts at each exclusion stage."
     },
     {
       "flag": "Submission bias unaddressed",
       "detail": "Leaderboard submissions are self-selected (only positive results submitted); the paper does not discuss how this biases observed architecture or LLM performance distributions."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-Bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "Primary benchmark analyzed; foundational paper for the entire study."
     },
     {
       "title": "Agentless: Demystifying LLM-Based Software Engineering Agents",
       "relevance": "Most-cited non-agentic approach; spawned multiple leaderboard extensions analyzed in detail."
     },
     {
       "title": "SWE-Agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "relevance": "Key emergent single-agent baseline (G6); analyzed across all three RQs."
     },
     {
       "title": "Large Language Model-Based Agents for Software Engineering: A Survey",
       "relevance": "Liu et al. taxonomy provides the pipeline phase framework used for RQ3."
     },
     {
       "title": "Are 'Solved Issues' in SWE-Bench Really Solved Correctly? An Empirical Study",
       "relevance": "Wang et al. finding of 6.2pp overstatement from patch overfitting, central to Section 4.1 discussion."
     },
     {
       "title": "Introducing SWE-Bench Verified",
       "relevance": "Describes construction criteria for the second leaderboard analyzed."
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "relevance": "Provides 14-failure-mode taxonomy used in Section 4.3's single vs. multi-agent debate."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "relevance": "G5 multi-agent scaffolded approach; one of the most-cited academic submissions on the leaderboard."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "relevance": "G6 open-source platform with multiple submissions; cited for single-agent architecture advocacy."
     },
     {
       "title": "Revisiting SWE-Bench: On the Importance of Data Quality for LLM-Based Code Models",
       "relevance": "Aleithan et al. patch quality analysis motivating discussion of evaluation reliability."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly informs AI practitioners on which LLMs, architectures, and product types are winning on the most-watched coding agent benchmark — immediately actionable."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the assumption that complex multi-agent architectures are superior — single-agent emergent systems (G6) are the largest and competitive group — and shows individual developers can match major tech companies."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI risk or safety concerns raised; focus is on benchmark performance and submitter characteristics."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Section 4.3 documents a real public dispute between Cognition (anti-multi-agent post) and Anthropic (pro-multi-agent post one day later), and highlights academia vs. industry evaluation standard misalignment."
     },
     "demo_ability": {
       "score": 1,
       "justification": "SWE-Bench leaderboard is public and readers can explore submissions, but the paper itself is analytical without a demo-able artifact."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Analyzes submissions from Anthropic, Google, Amazon, IBM, ByteDance, Meta, and Princeton, providing high name-recognition density even though the authors are from UPC Barcelona."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "44489690",
         "title": "Mercury: Ultra-fast language models based on diffusion",
         "points": 576,
         "comments": 242,
         "url": "https://news.ycombinator.com/item?id=44489690",
         "created_at": "2025-07-07T12:31:08Z"
       },
       {
         "hn_id": "44412427",
         "title": "Mercury: Ultra-Fast Language Models Based on Diffusion",
         "points": 10,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=44412427",
         "created_at": "2025-06-29T12:05:48Z"
       },
       {
         "hn_id": "44358841",
         "title": "Machine Mental Imagery: Empower Multimodal Reasoning with Latent Visual Tokens",
         "points": 7,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44358841",
         "created_at": "2025-06-23T18:52:55Z"
       },
       {
         "hn_id": "44101770",
         "title": "Effective Reinforcement Learning for Reasoning in Language Models",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44101770",
         "created_at": "2025-05-26T21:17:20Z"
       },
       {
         "hn_id": "44314613",
         "title": "Wanting to Be Understood Explains the Meta-Problem of Consciousness",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44314613",
         "created_at": "2025-06-19T01:16:41Z"
       },
       {
         "hn_id": "44304578",
         "title": "Serving Large Language Models on Huawei CloudMatrix384",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44304578",
         "created_at": "2025-06-17T22:18:43Z"
       },
       {
         "hn_id": "44009979",
         "title": "A Search for Planet Nine with IRAS and Akari Data",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44009979",
         "created_at": "2025-05-16T21:35:58Z"
       },
       {
         "hn_id": "46445614",
         "title": "Mechanical non-reciprocity programmed by shear jamming in soft composite solids",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46445614",
         "created_at": "2025-12-31T16:32:15Z"
       },
       {
         "hn_id": "44047429",
         "title": "Model Merging in Pre-Training of Large Language Models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44047429",
         "created_at": "2025-05-21T01:12:29Z"
       },
       {
         "hn_id": "42816449",
         "title": "Dissecting the NVIDIA Hopper Architecture through Microbenchmarking",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42816449",
         "created_at": "2025-01-24T20:02:41Z"
       }
     ],
     "top_points": 576,
     "total_points": 612,
     "total_comments": 244
   }
 }