ai-research-survey

scan.json (26208B)

---

 {
   "paper": {
     "title": "SWE-EVO: Benchmarking Coding Agents in Long-Horizon Software Evolution Scenarios",
     "authors": ["Minh Vu Thai Pham", "Tue Le", "Dung Nguyen Manh", "Huy Nhat Phan", "Nghi D. Q. Bui"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.18470"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SWE-EVO is a benchmark of 48 long-horizon software evolution tasks constructed from release notes of 7 open-source Python projects. Even the best model (GPT-5) resolves only 21% of SWE-EVO tasks versus 65% on SWE-Bench Verified, revealing a large capability gap for multi-file, multi-step coding tasks. The paper proposes Fix Rate as a soft metric capturing partial progress and provides trajectory-level failure analysis showing stronger models fail primarily on instruction following while weaker models fail on tool use and syntax errors.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository linked: https://github.com/SWE-EVO/SWE-EVO (Section 1, under title)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset (48 tasks) is released via the GitHub repository. The paper states it is constructed from public open-source repositories and is designed to be 'plug-and-play' for existing SWE agents."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper inherits execution environments from SWE-bench and SWE-gym (Section 3.1), and Table 6 shows each instance includes a Docker image field for the execution environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper itself. The GitHub link is given but no README or reproduction section is described in the text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2, 3, and 4 report only point estimates (e.g., '18.75% Resolved') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims about model performance (e.g., GPT-5 vs. others) but uses no statistical significance tests. Rankings are based solely on comparing raw percentages."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports performance differences with baseline context, e.g., 'GPT-5 achieves only 21% on SWE-EVO versus 65% on SWE-Bench Verified' (Section 1, Tables 2-3), providing magnitude of the gap."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has only 48 instances. The Limitations section acknowledges 'The 48-instance scale, while ensuring quality, limits statistical power for fine-grained comparisons' but does not justify why 48 was the resulting size beyond the filtering pipeline."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across any experimental runs. Each model-scaffold combination appears to be a single run."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "SWE-Bench Verified scores are included as comparison points for all models (Tables 2-3). Multiple models serve as baselines against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-5, o3, DeepSeek-R1, Kimi-K2, Qwen3-Coder — all 2025-era state-of-the-art models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates two input settings (release-note only vs. release-note + PR/issue context) as a form of ablation on input information (Tables 2-3), and analyzes performance vs. difficulty (PR count) in Section 4.3.2."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: Resolved Rate, Patch Apply Rate, and Fix Rate (Section 3.2, Tables 2-4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of agent outputs is conducted. Evaluation is entirely automated via test suites. The failure mode analysis uses LLM-as-a-judge (Section 4.3.1), not human judges."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper — the entire dataset is the test set. There is no train/test split relevant here."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by agent framework (OpenHands vs. SWE-agent), by input setting (Tables 2-3), and by difficulty level (Figure 7). Failure modes are broken down per model family (Figure 6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 provides detailed trajectory-level failure mode analysis with a taxonomy of 7 failure categories (Table 5, Figure 6), showing how different models fail differently."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The core finding is a negative result: even the best model solves only 21% of tasks. The paper also shows that providing PR/issue context yields only 'modest gains' (Section 4.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims GPT-5 achieves 21% on SWE-EVO vs. 65% on SWE-Bench Verified, which matches Table 2 (20.83% with SWE-agent). The claim about multi-step modifications spanning 21 files matches Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims PR count is a 'reliable proxy for instance difficulty' (Section 4.3.2) based on correlation, but this is presented as near-causal ('instances that aggregate many such changes are expected to require deeper reasoning'). The correlation could be confounded by repository-specific effects."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark covers only 7 Python repositories with 48 instances, yet the paper frames findings broadly as about 'autonomous software evolution' and 'long-horizon agent capabilities' (Section 5). The Limitations section acknowledges Python-only coverage but the title and abstract do not bound the claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the performance gap. For example, it does not consider whether the gap is due to context window limitations, task formulation differences (release notes vs. issues), or repository-specific effects rather than fundamental capability limitations."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures test-suite pass rates and frames this as evaluating 'software evolution capability.' It does not discuss whether passing tests on 48 tasks is an adequate proxy for the broader claim of autonomous software evolution capability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are listed in Section 4.1: 'GPT-5-pro-10-06, GPT-5-08-07, GPT-5-mini-08-07, GPT-5-nano-08-07', 'O3-2025-04-16', 'GPT-4.1-2025-04-14', 'GPT-4o-2024-11-20', 'Deepseek-R1-0528'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes problem statements at a high level and provides two examples (Appendix), but does not provide the actual system prompts or instructions given to the agent frameworks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: 'For OpenHands, we set a maximum of 100 iterations per instance. For SWE-Agent, we limit the number of LLM calls to 100.' Footnote 1 specifies 'medium reasoning effort' for GPT-5 and o3 models."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates existing third-party agent frameworks (OpenHands, SWE-agent) as black boxes. It cannot be expected to describe their internal scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents the three-stage construction pipeline: repository selection from SWE-bench/SWE-gym, candidate selection by version tags, and execution-based filtering with specific criteria at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A Limitations paragraph is present in Section 5 (Conclusion): 'SWE-EVO currently covers only Python projects and relies on release notes as specifications...'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section identifies specific threats: Python-only coverage, reliance on release notes (missing security patches and performance optimizations), and the 48-instance scale limiting statistical power."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section states specific things not covered: 'may not capture all evolution scenarios (e.g., security patches, performance optimizations without explicit notes)' and calls for expanding language coverage and instance count."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is released via GitHub (https://github.com/SWE-EVO/SWE-EVO) with task instances including patches, test patches, and problem statements (Table 6)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes three stages of data collection: sampling from SWE-bench and SWE-gym, filtering to version-tag commits, and execution-based validation."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is standard public repositories via SWE-bench."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents the full pipeline from seed pool (SWE-bench + SWE-gym) through version-tag filtering to execution-based filtering, resulting in 48 instances. Each stage's criteria are described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or disclosure section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: FPT Software AI Center and University of Melbourne."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Authors are from FPT Software AI Center, a software company. No funding disclosure is present, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the 11 models evaluated, despite this being critical for assessing whether models may have seen the benchmark repositories."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the open-source repositories (scikit-learn, pydantic, etc.) and their release notes are in model training data. These are extremely popular repos that are very likely in training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is constructed from well-known open-source projects (scikit-learn, pydantic, dask, etc.) whose release notes and patches are publicly available on GitHub. The paper does not discuss contamination risk despite this being a central concern."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or wall-clock times reported for running the 11 models × 2 scaffolds × 48 tasks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget stated. The paper evaluates 11 models across 2 frameworks on 48 tasks (1,056+ runs) but does not quantify the compute required."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multiple-seed runs reported. Each model-scaffold pair appears to produce a single set of results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many times each experiment was run. Results appear to be single-run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. The paper uses default agent configurations without discussing whether alternatives were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'medium' reasoning effort for GPT-5 and o3 models (footnote 1) but does not justify this choice or compare against other settings."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own benchmark but do not acknowledge potential author-evaluation bias in benchmark design or task selection."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Different models have vastly different compute costs (GPT-5-pro vs GPT-5-nano, proprietary vs open-source) but performance is never compared at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 discusses construct validity by comparing SWE-EVO features against SWE-Bench (Figure 4), analyzing PR count as difficulty proxy (Section 4.3.2), and showing intuitive scaling behavior that validates the benchmark measures meaningful capability."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates each model across both OpenHands and SWE-agent scaffolds (Tables 2-4), allowing readers to see scaffold effects. Results are reported separately per scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses release notes from mature open-source projects (scikit-learn, pydantic, dask). The paper does not discuss whether these releases predate model training, despite this being critical."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the problem statement (release notes with PR/issue links) provides information that leaks the solution."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is constructed from SWE-bench seed data. No discussion of whether SWE-EVO instances overlap with SWE-bench training examples used by models post-trained on SWE-bench data (e.g., SWE-RL, DeepSWE)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used despite the benchmark being derived from public repositories and release notes."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-5 achieves only 21% on SWE-EVO versus 65% on SWE-Bench Verified, revealing a significant capability gap for long-horizon software evolution tasks.",
       "evidence": "Table 2 shows GPT-5-08-07 resolves 20.83% (SWE-agent) and 18.75% (OpenHands) on SWE-EVO, compared to 65% on SWE-Bench Verified.",
       "supported": "strong"
     },
     {
       "claim": "Model performance on SWE-EVO exhibits intuitive scaling: larger models consistently outperform smaller variants.",
       "evidence": "Tables 2-3 show GPT-5 > GPT-5-mini > GPT-5-nano consistently across both scaffolds (e.g., 20.83% > 10.42% > 4.17% with SWE-agent).",
       "supported": "strong"
     },
     {
       "claim": "Providing PR/issue context yields measurable but modest gains over release-note-only input.",
       "evidence": "Comparing Tables 2 and 3: GPT-5 goes from 16.67% to 20.83% (SWE-agent) with context. Most models show small improvements.",
       "supported": "moderate"
     },
     {
       "claim": "Stronger models fail primarily on instruction following while weaker models struggle with tool use and syntax errors.",
       "evidence": "Figure 6 and Section 4.3.1 show GPT-5 failures are >60% instruction following, while GPT-5-nano has substantial tool-use and syntax errors.",
       "supported": "moderate"
     },
     {
       "claim": "PR count is a reliable proxy for instance difficulty.",
       "evidence": "Figure 7a shows monotonic trend: easier instances (frequently solved) average 1.67 PRs, hardest average 14.84 PRs (Section 4.3.2).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Tiny benchmark size",
       "detail": "Only 48 instances across 7 repositories. With such small N, individual instance variation dominates. A single instance changing outcome could shift resolved rates by ~2 percentage points. The paper acknowledges this in limitations."
     },
     {
       "flag": "No contamination analysis",
       "detail": "The benchmark uses well-known open-source repos (scikit-learn, pydantic, dask) whose release notes, commits, and patches are publicly available on GitHub and likely in LLM training data. Several evaluated models (DeepSWE, SWE-RL) were explicitly trained on SWE-bench data, from which SWE-EVO is derived. No contamination analysis is performed."
     },
     {
       "flag": "LLM-as-a-judge for failure analysis",
       "detail": "Section 4.3.1 uses GPT-5-mini as an automated judge for failure mode classification. No validation of judge accuracy against human labels is provided."
     },
     {
       "flag": "No variance or repeat runs",
       "detail": "All results appear to be single-run. With stochastic LLM outputs and only 48 instances, single-run results are unreliable. A different random seed could change which instances are solved."
     },
     {
       "flag": "Company affiliation without disclosure",
       "detail": "Authors are from FPT Software AI Center. No funding, competing interests, or financial interests declarations are provided."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Foundational benchmark for evaluating coding agents on real-world GitHub issues; SWE-EVO extends this to multi-step evolution tasks."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "One of two agent frameworks used for evaluation in SWE-EVO experiments."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Second agent framework used for evaluation; emphasizes importance of agent-computer interfaces."
     },
     {
       "title": "Agentless: Demystifying LLM-based Software Engineering Agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Simpler localization-repair approach that challenges complex agent scaffolding, relevant to agent architecture evaluation."
     },
     {
       "title": "SWE-RL: Advancing LLM Reasoning via Reinforcement Learning on Open Software Evolution",
       "authors": ["Yuxiang Wei"],
       "year": 2025,
       "arxiv_id": "2502.18449",
       "relevance": "RL-based post-training for software engineering; raises contamination concerns when evaluated on SWE-EVO."
     },
     {
       "title": "AgileCoder: Dynamic Collaborative Agents for Software Development Based on Agile Methodology",
       "authors": ["Minh Huynh Nguyen", "Thang Phan Chau"],
       "year": 2025,
       "relevance": "Multi-agent framework incorporating agile methodology for software development."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "relevance": "Autonomous coding agent combining LLMs with AST-based code search for program repair."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["Unknown"],
       "year": 2023,
       "relevance": "Multi-agent framework for software development through role-based communication."
     },
     {
       "title": "SWE-bench Pro: Can AI Agents Solve Long-Horizon Software Engineering Tasks?",
       "authors": ["Scale AI"],
       "year": 2025,
       "arxiv_id": "2509.16941",
       "relevance": "Concurrent benchmark targeting enterprise-level complexity; directly related to long-horizon evaluation."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark paper; foundational code generation evaluation that SWE-EVO aims to extend beyond."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Code generation benchmark with complex instructions, relevant to benchmark design methodology."
     },
     {
       "title": "SWE-Synth: Synthesizing Verifiable Bug-Fix Data to Enable Large Language Models in Resolving Real-World Bugs",
       "authors": ["Minh VT Pham", "Huy N Phan"],
       "year": 2025,
       "arxiv_id": "2504.14757",
       "relevance": "Synthetic bug-fix data generation for training coding agents; by same research group as SWE-EVO."
     }
   ]
 }