ai-research-survey

scan.json (25024B)

---

 {
   "paper": {
     "title": "LIVE-SWE-AGENT: Can Software Engineering Agents Self-Evolve on the Fly?",
     "authors": [
       "Chunqiu Steven Xia",
       "Zhe Wang",
       "Yan Yang",
       "Yuxiang Wei",
       "Lingming Zhang"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2511.13646"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/OpenAutoCoder/live-swe-agent. Paper states it 'has been publicly available.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Uses publicly available benchmarks: SWE-bench Verified, SWE-Bench Pro, and SWE-bench Multilingual. No proprietary data collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided in the paper. Only mentions building on mini-SWE-agent (~100 lines of code)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions in the paper. The GitHub link exists but the paper itself does not contain a 'Reproducing Results' section or specific commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '77.4%', '45.8%') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'outperforming all existing software agents' are based on comparing raw percentages without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Reports improvements with baseline context throughout, e.g., 'improving the resolve rate by 8.3 percentage points compared to previous best approach' (Section 4.1), and Table 5 shows relative improvements like '↑22.6%'."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The ablation uses 50 'randomly selected' problems with no justification for why 50 is sufficient. The 60-problem subset from prior work is also used without power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Paper explicitly states 'we sample one patch per issue' (Section 3). All results are single-run with no variance, standard deviation, or spread measures reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Compares against mini-SWE-agent, SWE-agent, and self-evolving agents (SICA, DGM, HGM) in Tables 1-3 and Figure 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include state-of-the-art systems: Claude Sonnet 4.5, GPT-5, Gemini 3 Pro, and recent agents like DGM (2025), HGM (2025), OpenHands, Trae."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 ablates tool creation (w/o tool creation) and reflection (w/o reflection), showing contribution of each component on 50 problems."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Reports both resolve rate (%) and average cost ($) per issue across experiments (Tables 1, 3, 6). Table 4 also reports number of tools created."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via SWE-bench test suites (pass/fail). No human evaluation of patch quality, maintainability, or correctness beyond tests."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "SWE-bench Verified, SWE-Bench Pro, and SWE-bench Multilingual are standard held-out benchmarks not used for development of the method."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "No per-repository or per-difficulty breakdown of resolve rates. SWE-Bench Pro spans 11 repositories and 4 languages but only aggregate results are reported in Table 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses GPT-5-Nano failing with LIVE-SWE-AGENT: 'GPT-5-Nano fails to understand the goal of creating custom tools and is often stuck in a loop.' Also discusses when tool creation doesn't help."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports GPT-5-Nano performance drops 68.2% with LIVE-SWE-AGENT vs mini-SWE-agent, and GPT-5-Mini drops 3.3%. These negative results are discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 77.4% on Verified and 45.8% on Pro are supported by Tables 1 and 3. Claim of outperforming all existing agents is supported by Figure 1 comparisons."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claim that tool creation improves performance is supported by controlled ablation in Table 4 (same 50 problems, same LLM, varying only the tool creation/reflection components)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Title says 'Software Engineering Agents' broadly. Claims 'outperforming all existing software agents' but tests only on SWE-bench family benchmarks (issue resolution). Section 4.4 speculates about applications to testing, vulnerability detection, and software synthesis without evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for performance gains. Could the improvement be from the additional reflection prompt rather than tool creation itself? The ablation partially addresses this but no alternative explanations are considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Claims are stated at the level of measurements: 'resolve rate' on specific benchmarks. No overclaiming that resolve rate equals general software engineering ability (though Section 4.4 speculates about broader applications, these are framed as future work)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specifies 'claude-sonnet-4-5-20250929' with exact API identifier (Section 3). Temperature settings also specified per model family (Appendix A)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full initial prompt and feedback message provided in Appendix D (Figures 7 and 8), including the complete tool creation instructions and reflection message."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 and Appendix A report: max step limit 250, max cost $3/issue, temperature 0.0 for Anthropic models, temperature 1 for GPT and Gemini models."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the scaffold in detail: agent loop, tool creation mechanism, reflection prompt, custom tool synthesis process. Figure 2 provides an overview diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Uses standard benchmarks without modification. SWE-bench Verified (500 problems), SWE-Bench Pro (731 public problems), and a 50-problem random subset of SWE-bench Multilingual are clearly specified."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. Section 4.4 is 'Discussion and Future Work' which discusses extensions and applications but does not substantively discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity discussed. No mention of internal or external validity concerns."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries stated. The paper does not state what the results do NOT show or which settings are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Agent trajectories, tool scripts generated, and detailed per-problem results are not released. Only aggregate results are shown. The ablation problem lists are provided (Appendix C) but not the raw outputs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are standard benchmarks with well-documented collection procedures: SWE-bench Verified (validated by human developers), SWE-Bench Pro (Scale AI), SWE-bench Multilingual."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline is clear: issue description → agent with initial prompt → iterative tool creation/usage loop → patch submission → automated evaluation via SWE-bench harness."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source disclosed anywhere in the paper. No acknowledgments section with grant information."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors' affiliations clearly stated: University of Illinois Urbana-Champaign. One author noted as research intern."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed. The authors use commercial APIs (Anthropic, OpenAI, Google) but no disclosure of whether API credits were provided."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Uses Claude 4.5 Sonnet, GPT-5 variants, Gemini 3 Pro without stating any of their training data cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLMs' training data includes SWE-bench problems or their solutions from GitHub."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench problems come from public GitHub issues that could be in LLM training data. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1, 3, and 6 report average dollar cost per issue for each configuration. E.g., LIVE-SWE-AGENT with Claude 4.5 Sonnet costs $0.68/issue on Verified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Maximum cost of $3 per issue and maximum step limit of 250 stated (Section 3). Per-issue average costs reported. Table 2 also contrasts offline training costs (DGM: 1231 hours vs LIVE-SWE-AGENT: 0 hours)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Single attempt per problem: 'we sample one patch per issue' (Section 3). No seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly states 'sample one patch per issue' (Section 3), making clear these are single-run results."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No description of how the reflection prompt or tool creation instructions were developed. No hyperparameter search budget reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The final prompt design is presented without explaining what alternatives were tried or how the configuration was selected."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors evaluate their own system against baselines including their own re-use of mini-SWE-agent results. No acknowledgment of self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Cost is reported alongside performance in Tables 1, 3, 6. The paper notes LIVE-SWE-AGENT achieves improvements with 'only a minimal increase in cost' and even 'slight cost savings' in some cases (Section 4.1)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether SWE-bench resolve rate actually measures real-world software engineering capability. The paper uses the benchmarks without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper controls for scaffold: Table 1 compares mini-SWE-agent vs LIVE-SWE-AGENT with the same LLMs. Table 3 uses same LLM (Claude 4.5 Sonnet) for SWE-agent vs LIVE-SWE-AGENT comparison. The paper's core contribution IS the scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench problems are from public GitHub issues that predate model training. No discussion of temporal leakage risk."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information not available in real usage scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data includes solutions to the same GitHub issues used in SWE-bench."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method applied."
       }
     }
   },
   "claims": [
     {
       "claim": "LIVE-SWE-AGENT achieves 77.4% resolve rate on SWE-bench Verified without test-time scaling, outperforming all existing agents.",
       "evidence": "Table 1 and Figure 1 show 77.4% with Gemini 3 Pro. Figure 1 compares against open and proprietary agents.",
       "supported": "strong"
     },
     {
       "claim": "LIVE-SWE-AGENT achieves 45.8% on SWE-Bench Pro, the best reported result.",
       "evidence": "Table 3 shows 45.8% vs SWE-agent's 43.6% with the same LLM (Claude 4.5 Sonnet). Figure 1 confirms top position.",
       "supported": "strong"
     },
     {
       "claim": "On-the-fly tool creation improves performance over the base scaffold with minimal cost overhead.",
       "evidence": "Table 4 ablation on 50 problems: 62% (no tool creation) → 64% (no reflection) → 76% (full). Tables 1, 3, 6 show marginal cost increases.",
       "supported": "moderate"
     },
     {
       "claim": "LIVE-SWE-AGENT outperforms offline self-evolving agents (DGM, HGM, SICA) while requiring zero offline training cost.",
       "evidence": "Table 2: 65.0% vs DGM's 53.3% on the 60-problem subset. DGM costs 1231 hours offline; LIVE-SWE-AGENT costs 0.",
       "supported": "moderate"
     },
     {
       "claim": "Stronger LLMs benefit more from on-the-fly tool creation.",
       "evidence": "Table 5: GPT-5-Nano drops 68.2%, GPT-5-Mini drops 3.3%, while Claude 4.5 Sonnet gains 22.6% with LIVE-SWE-AGENT.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LIVE-SWE-AGENT enables LLM-based software agents to create and modify their own tools at runtime while solving issues, achieving state-of-the-art 77.4% on SWE-bench Verified and 45.8% on SWE-Bench Pro. The approach requires only minimal modifications to existing agent scaffolds (initial prompt + reflection message) with no offline training. Ablations show tool creation and step-level reflection both contribute to improvements, but weaker LLMs (GPT-5-Nano) lack the reasoning capability to benefit from on-the-fly tool synthesis.",
   "red_flags": [
     {
       "flag": "No statistical tests for any comparison",
       "detail": "All claims of 'outperforming' are based on comparing raw percentages without significance tests. On a 500-problem benchmark, a 2-3% difference may not be statistically significant."
     },
     {
       "flag": "Single-run results",
       "detail": "All results are from a single attempt per problem with no variance estimates. LLM outputs are stochastic, so single-run results may not be stable, especially with temperature=1 for GPT and Gemini models."
     },
     {
       "flag": "Small ablation sample without justification",
       "detail": "Ablation experiments use only 50 randomly selected problems (10% of SWE-bench Verified) with no power analysis or justification for sample size adequacy."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity discussion. Section 4.4 discusses future work and applications but does not acknowledge weaknesses."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "SWE-bench problems come from public GitHub issues. Models like GPT-5 and Gemini 3 Pro may have seen both the issues and their solutions during training. This is not discussed."
     },
     {
       "flag": "Leaderboard comparison mixes scaffolds",
       "detail": "Figure 1 compares LIVE-SWE-AGENT against proprietary systems with different scaffolds (Claude Code, OpenAI's custom setup). While the paper's controlled experiments (Tables 1, 3) are fair, the leaderboard claims conflate scaffold and model differences."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "relevance": "Primary baseline agent scaffold; LIVE-SWE-AGENT builds on mini-SWE-agent from this work."
     },
     {
       "title": "OpenHands: An open platform for AI software developers as generalist agents",
       "authors": ["Xingyao Wang"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "Major open-source agentic framework for software engineering, compared in leaderboard."
     },
     {
       "title": "Agentless: Demystifying LLM-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Contrasting approach arguing complex agent scaffolds are unnecessary; uses predefined workflows instead."
     },
     {
       "title": "Darwin Godel Machine: Open-ended evolution of self-improving agents",
       "authors": ["Jenny Zhang", "Shengran Hu", "Cong Lu", "Robert Lange", "Jeff Clune"],
       "year": 2025,
       "arxiv_id": "2505.22954",
       "relevance": "Key baseline for self-improving agents; requires costly offline training ($22K per run) compared to LIVE-SWE-AGENT's zero offline cost."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Foundational benchmark for evaluating software engineering agents on real-world issues."
     },
     {
       "title": "SWE-RL: Advancing LLM reasoning via reinforcement learning on open software evolution",
       "authors": ["Yuxiang Wei"],
       "year": 2025,
       "arxiv_id": "2502.18449",
       "relevance": "RL-based approach to improving LLMs for software engineering tasks."
     },
     {
       "title": "Conversational automated program repair",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2301.13246",
       "relevance": "Early work on interactive LLM-based bug fixing (ChatRepair), precursor to agentic approaches."
     },
     {
       "title": "Huxley-Gödel Machine: Human-level coding agent development by an approximation of the optimal self-improving machine",
       "authors": ["Wenyi Wang"],
       "year": 2025,
       "relevance": "Self-improving agent baseline requiring 512 hours offline training; outperformed by LIVE-SWE-AGENT."
     },
     {
       "title": "SICA: A self-improving coding agent",
       "authors": ["Maxime Robeyns", "Martin Szummer", "Laurence Aitchison"],
       "year": 2025,
       "relevance": "Self-improving agent baseline that enters infinite loops; outperformed by LIVE-SWE-AGENT."
     },
     {
       "title": "Large language models as tool makers",
       "authors": ["Tianle Cai", "Xuezhi Wang", "Tengyu Ma", "Xinyun Chen", "Denny Zhou"],
       "year": 2024,
       "relevance": "Prior work on LLMs creating tools for general reasoning, though not targeting real-world software engineering."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex/HumanEval paper foundational to code generation evaluation."
     },
     {
       "title": "AutoCodeRover: Autonomous program improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "relevance": "Software engineering agent using code search and analysis for autonomous bug fixing."
     }
   ]
 }