ai-research-survey

scan.json (26942B)

---

 {
   "paper": {
     "title": "SWE-Bench Pro: Can AI Agents Solve Long-Horizon Software Engineering Tasks?",
     "authors": [
       "Xiang Deng",
       "Jeff Da",
       "Edwin Pan",
       "Yannis Yiming He",
       "Charles Ide",
       "Kanak Garg",
       "Niklas Lauffer",
       "Andrew Park",
       "Nitin Pasari",
       "Chetan Rane",
       "Karmini Sampath",
       "Maya Krishnan",
       "Srivatsa Kundurthy",
       "Sean Hendryx",
       "Zifan Wang",
       "Vijay Bharadwaj",
       "Jeff Holm",
       "Raja Aluri",
       "Chen Bo Calvin Zhang",
       "Noah Jacobson",
       "Bing Liu",
       "Brad Kenstler"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.16941"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SWE-Bench Pro introduces 1,865 human-verified benchmark problems from 41 repositories spanning multiple languages, with tasks requiring multi-file modifications averaging 107.4 lines across 4.1 files. Top models (Claude Sonnet 4.5, GPT-5) achieve only ~43% on the public set and under 18% on the commercial set, compared to 70%+ on SWE-Bench Verified. The benchmark uses GPL-licensed and proprietary codebases to resist contamination. Failure analysis reveals that even frontier models struggle with semantic understanding and multi-file coordination.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository URL: https://github.com/scaleapi/SWE-bench_Pro-os and a HuggingFace dataset link: https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The public set of 731 instances is released on HuggingFace. The held-out (858) and commercial (276) sets are kept private by design."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 and 4.3 describe Docker-based containerized environments for each task. 'All environments will be released as pre-built docker images to ensure that they are fully reproducible.'"
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code and data are released, the paper does not include step-by-step reproduction instructions for replicating the evaluation results (e.g., specific commands to run)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, and 5 report only point estimates (e.g., '43.6%') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares model performance (e.g., 'Claude Sonnet 4.5 achieves 43.6% vs GPT-5 at 41.8%') without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute resolve rates and contextualizes them against SWE-Bench Verified (e.g., '23% on SWE-Bench Pro compared to over 70% on SWE-Bench Verified'), providing magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 1,865 problems total (731 public), but there is no justification for why this size is sufficient or any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as single-run Pass@1 scores without any variance, standard deviation, or multi-run statistics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple models are compared including frontier (Claude Sonnet 4.5, GPT-5, Claude Opus 4.1), mid-tier (Claude Haiku 4.5, Gemini 2.5), and open-source (Qwen3 32B, GPT-OSS 120B) models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include Claude Sonnet 4.5, GPT-5, and other frontier models as of September 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 presents an ablation removing human augmentations (requirements and interface), showing performance drops from 25.9% to 8.4% for GPT-5 and 22.7% to 8.2% for Opus 4.1."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only Pass@1 (resolve rate) is reported. No other metrics such as partial solve rates, code quality, or cost-efficiency are measured."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark construction involves human verification, but evaluation of model outputs is entirely automated via test suites. No human evaluation of generated patches is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark has a held-out set of 858 problems 'to test for overfitting in the future,' and the commercial set is also private. Results are reported on the public and commercial sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 provides breakdowns by programming language, repository, file count, and lines of code. Table 4 provides detailed failure mode breakdowns per model."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 provides detailed trajectory failure mode analysis with categories (wrong solution, tool-use errors, syntax errors, etc.) quantified per model in Table 4."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that Agentless scaffold 'has difficulty in multi-file editing, thus, produces low evaluation scores.' The overall low performance of all models (<45%) is itself a negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims performance 'below 45% (Pass@1)' which matches Table 1 (highest is 43.6%). The contamination-resistant design claim is supported by the GPL/commercial strategy described in Sections 3-4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation in Table 3 makes a causal claim about human augmentations improving performance, supported by a controlled single-variable manipulation (with vs. without augmentations)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title asks about 'Long-Horizon Software Engineering Tasks' broadly, but the benchmark covers only Python, JavaScript, TypeScript, and Go. Section 7.1 acknowledges limited language coverage but the title and abstract still frame results broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why models fail (e.g., could the test suites be overly strict? could the human augmentations favor certain solution styles?). The failure mode analysis categorizes errors but doesn't consider confounds."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures pass/fail on test suites as a proxy for 'solving software engineering tasks,' but does not discuss the gap between test-suite passing and actual issue resolution quality. Section 7.1 briefly notes 'real software engineering tasks may have a variety of correct solutions' but doesn't discuss this as a proxy limitation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names (e.g., 'Claude Sonnet 4.5', 'OpenAI GPT-5 (High)', 'Kimi K2 Instruct') without specific version IDs or snapshot dates. The paper states 'latest versions as of September 18th, 2025' but does not provide API version strings."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'We use the same prompt for all models, which is the default prompt from [16]' but does not include the actual prompt text. The failure mode analysis prompt is provided in Appendix C but the evaluation prompt is not."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for the model evaluations. The only setting mentioned is 'maximum of 50 turns' and cost limit of $2."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We use the SWE-Agent scaffold' and references [16] but does not describe the scaffold's internals (tool descriptions, retry logic, context management). The tools are listed in Appendix C as part of the LLM-as-judge prompt, not the evaluation scaffold."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the full data creation pipeline: commit pair identification, test patch extraction, human augmentation process, environment construction, and test verification with clear filtering criteria."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 is titled 'Limitations' and discusses limited language coverage and dependency on test suites."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 identifies specific threats: limited language coverage ('Java, C++, and Rust are underrepresented') and that test suites may reject valid alternative solutions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While limitations are mentioned, the paper does not explicitly state what the results do NOT show. For example, it does not state that the benchmark cannot assess code quality, maintainability, or real-world development workflow integration."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The public set of 731 instances is released on HuggingFace, including problem statements, test patches, and gold patches."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the full data collection procedure: identifying commit pairs, extracting test patches, the human augmentation process, environment construction, and verification."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the professional software engineers who created environments and augmented tasks were recruited, nor how the startup companies for commercial repositories were selected."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 4.1-4.3 document the pipeline from commit scraping → task description creation → environment construction → environment verification → test verification, with filtering criteria at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. All authors are from Scale AI, a commercial entity, but no funding acknowledgments section exists."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are clearly listed as affiliated with Scale AI."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Scale AI is a commercial company that sells AI evaluation and data services. They have a direct financial interest in benchmarks being adopted and in demonstrating limitations of AI models that their services could help address."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. Scale AI is a for-profit company that sells benchmark evaluation as a service, creating a potential financial interest in benchmark adoption."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models. The paper only notes 'latest versions as of September 18th, 2025.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses contamination risk and designs the benchmark to mitigate it through GPL licenses and commercial codebases (Section 1, Section 3.1), though no empirical overlap analysis is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination resistance is a core contribution. The paper uses GPL-licensed repos (legal barrier to training inclusion) and proprietary commercial repos to reduce contamination risk. Section 3.1 discusses this extensively."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "A cost cap of $2 per instance is mentioned for the analysis runs (Table 5), but actual inference costs per model are not reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states 'Models are hosted on a single node, with 8 H100 Nvidia GPUs' for open-source models. A cost limit of $2 per instance and 50-turn limit are specified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be single-run Pass@1 scores with no mention of multiple seeds or seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the number of runs. Results appear to be single runs given they report Pass@1 without variance."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The paper uses default SWE-Agent settings without discussing whether alternatives were explored."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses default SWE-Agent configuration without justifying why this is optimal or comparing alternative configurations."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Scale AI created the benchmark and evaluated models on it. No discussion of author-evaluation bias or potential for task design favoring certain model behaviors."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 uses a cost cap of $2, but no performance-vs-compute curves are shown. Table 1 vs Table 5 show different results under different cost constraints but this is not explicitly analyzed as a compute budget comparison."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether pass/fail on test suites actually measures 'software engineering capability.' Section 7.1 notes test-suite dependency but does not analyze construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All models are evaluated using the same SWE-Agent scaffold, which is good practice. However, the paper notes Agentless performs poorly but doesn't systematically evaluate scaffold confounds. The paper mentions using 'a unified scaffold' but does not discuss how scaffold choice affects results beyond dismissing Agentless."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses temporal leakage through its contamination-resistant design: using GPL-licensed repos (legal barrier), commercial proprietary repos, and a held-out set. Section 3.1 discusses this explicitly."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the human augmentations (requirements, interface) provide information that would not be available in real-world usage, potentially inflating results."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether problems from the same repository share structural similarities that could inflate aggregate performance metrics."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark uses GPL licensing as a legal contamination prevention method and proprietary commercial repos as a structural prevention method. A held-out set is reserved for future overfitting detection."
       }
     }
   },
   "claims": [
     {
       "claim": "Top models achieve below 45% Pass@1 on SWE-Bench Pro public set",
       "evidence": "Table 1: Claude Sonnet 4.5 at 43.6%, Claude Sonnet 4 at 42.7%, GPT-5 at 41.8%",
       "supported": "strong"
     },
     {
       "claim": "SWE-Bench Pro is significantly harder than SWE-Bench Verified, with top models scoring 23% vs 70%+",
       "evidence": "Section 8 states '23% success rate on SWE-Bench Pro compared to over 70% on benchmarks like SWE-Bench Verified'",
       "supported": "moderate"
     },
     {
       "claim": "Human augmentations (requirements, interface) substantially improve model performance and reduce false negatives",
       "evidence": "Table 3: GPT-5 drops from 25.9% to 8.4%, Opus 4.1 drops from 22.7% to 8.2% without augmentations",
       "supported": "strong"
     },
     {
       "claim": "GPL licensing and commercial repositories make the benchmark contamination-resistant",
       "evidence": "Section 3.1 argues GPL licenses create legal barriers to training inclusion, and commercial repos are private. No empirical contamination analysis is provided.",
       "supported": "moderate"
     },
     {
       "claim": "Performance degrades sharply with increasing file count, and frontier models maintain performance better than smaller models on complex tasks",
       "evidence": "Figure 3 shows performance-vs-file-count curves with widening gaps between frontier and open-source models beyond 3 files",
       "supported": "moderate"
     },
     {
       "claim": "Commercial set is significantly harder than public set, with best models scoring under 18%",
       "evidence": "Table 2: Claude Opus 4.1 at 17.8% on commercial vs Table 5 showing 22.7% on public (with cost cap). Different models/settings make comparison imprecise.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own benchmark",
       "detail": "Scale AI created SWE-Bench Pro and evaluates all models on it. As a company selling AI evaluation services, Scale has a financial interest in benchmarks being widely adopted. No independent evaluation or self-comparison bias acknowledgment."
     },
     {
       "flag": "No variance or multi-run statistics",
       "detail": "All results appear to be single-run Pass@1 with no error bars, confidence intervals, or multi-seed evaluation. LLM outputs are stochastic, so single-run results may not be stable."
     },
     {
       "flag": "Inconsistent comparison across tables",
       "detail": "Table 1 and Table 5 report different results for the same models on the same public set due to different cost/turn constraints ($2 cap vs uncapped), making it unclear which is the authoritative result. The conclusion cites 23% (Table 5) while the abstract claims below 45% (Table 1)."
     },
     {
       "flag": "LLM-as-judge for failure analysis",
       "detail": "Failure mode analysis uses GPT-5 as judge with only 87% alignment with human categorization cited from prior work. No validation of the judge's accuracy on SWE-Bench Pro's specific failure patterns."
     },
     {
       "flag": "GPL contamination argument is weak",
       "detail": "The claim that GPL licensing prevents training inclusion is a legal argument, not an empirical one. GPL code is widely available on GitHub and could be scraped regardless of license terms. No empirical contamination testing was performed."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig", "S. Yao", "K. Pei", "O. Press", "K. R. Narasimhan"],
       "year": 2024,
       "relevance": "Original SWE-bench benchmark that SWE-Bench Pro builds upon."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["J. Yang", "C. E. Jimenez", "A. Wettig", "K. Lieret", "S. Yao", "K. Narasimhan", "O. Press"],
       "year": 2024,
       "relevance": "SWE-Agent scaffold used for all evaluations in this paper."
     },
     {
       "title": "Agentless: Demystifying LLM-Based Software Engineering Agents",
       "authors": ["C. S. Xia", "Y. Deng", "S. Dunn", "L. Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Alternative scaffold evaluated but found to perform poorly on multi-file tasks."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark, foundational code generation evaluation."
     },
     {
       "title": "Multi-SWE-bench: A Multilingual Benchmark for Issue Resolving",
       "authors": ["D. Zan", "Z. Huang", "W. Liu"],
       "year": 2024,
       "arxiv_id": "2404.02605",
       "relevance": "Extended SWE-bench to multiple programming languages."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Y. Zhang", "H. Ruan", "Z. Fan", "A. Roychoudhury"],
       "year": 2024,
       "relevance": "SWE agent using AST-based code search, evaluated on SWE-bench."
     },
     {
       "title": "SWE-bench+: Enhanced Coding Benchmark for LLMs",
       "authors": ["R. Aleithan"],
       "year": 2024,
       "arxiv_id": "2410.06992",
       "relevance": "Enhanced version of SWE-bench addressing known limitations."
     },
     {
       "title": "SWE-bench Goes Live!",
       "authors": ["C. Zhang"],
       "year": 2025,
       "arxiv_id": "2505.23419",
       "relevance": "Live version of SWE-bench addressing contamination through temporal freshness."
     },
     {
       "title": "Agent-RLVR: Training Software Engineering Agents via Guidance and Environment Rewards",
       "authors": ["J. Da", "C. J. Wang", "X. Deng"],
       "year": 2025,
       "arxiv_id": "2506.11425",
       "relevance": "RL training approach for SWE agents using benchmark instances."
     },
     {
       "title": "A Careful Examination of Large Language Model Performance on Grade School Arithmetic",
       "authors": ["H. Zhang", "J. Da", "D. Lee"],
       "year": 2024,
       "arxiv_id": "2405.00332",
       "relevance": "Demonstrates contamination risks in LLM benchmarks."
     },
     {
       "title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models",
       "authors": ["C. Deng", "Y. Zhao", "X. Tang", "M. Gerstein", "A. Cohan"],
       "year": 2024,
       "relevance": "Study on data contamination in LLM benchmarks."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["J. Austin", "A. Odena", "M. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "MBPP benchmark for code generation evaluation."
     }
   ]
 }