ai-research-survey

scan.json (26926B)

---

 {
   "paper": {
     "title": "SWT-Bench: Testing and Validating Real-World Bug-Fixes with Code Agents",
     "authors": ["Niels Mündler", "Mark Niklas Müller", "Jingxuan He", "Martin Vechev"],
     "year": 2024,
     "venue": "Neural Information Processing Systems",
     "arxiv_id": "2406.12952",
     "doi": "10.52202/079017-2601"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'We release all data and code at github.com/logic-star-ai/SWT-Bench.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset is released alongside the code at the provided GitHub URL. The underlying data is based on publicly available SWE-Bench data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is described in the paper itself. Evaluation uses Docker containers but environment specifications are not detailed in the text."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include step-by-step reproduction instructions. While code is released, no README-level reproduction guide is provided in the paper text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Tables 2-6 report point estimates only with no confidence intervals or error bars. Table 8 (temperature ablation) shows 95% CIs for n=25, but this is an ablation, not the main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports statistical significance: 'This stronger performance is significant at p < 0.1%' for SWE-AGENT+ vs LIBRO (§5.2), and p-values for contamination analysis (p≈37%, Table 7) and overlap analysis (Table 6)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported as absolute percentages with baselines for context (e.g., 'almost 3x increase in success rate to 9.4%', 'more than doubles the precision of SWE-AGENT to 47.8%'). Tables provide full context for comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of SWT-BENCH-LITE (276 instances) over full SWT-BENCH is stated as 'due to budget constraints' but no power analysis or sample size justification is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Tables 2-4) are single-run results at temperature 0. No variance across runs is reported for the primary experiments. Table 8 shows variance for the temperature ablation only."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are compared: ZEROSHOT, ZEROSHOTPLUS, LIBRO, PASS@5, AUTOCODEROVER, AIDER, SWE-AGENT, and SWE-AGENT+ (Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary Code Agents (SWE-Agent, AutoCodeRover, Aider) and the state-of-the-art test generation method LIBRO, all from 2023-2024."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablations are provided: number of LIBRO samples (Fig. 8a), interaction rounds for agents (Fig. 8b), temperature (Table 8), effect of providing code patches/test files (Table 5), and SWE-AGENT vs SWE-AGENT+ (test execution ablation)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are defined and reported: Success rate (S), Change Coverage (ΔC), Well-formedness (W), F→P rate, F→× rate, P→P rate (§3.3, Tables 2-3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated tests is performed. All evaluation is automated via test execution against golden patches."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "SWT-BENCH-LITE is used for evaluation. The golden patches serve as held-out ground truth not provided to the methods being evaluated. No tuning is performed on the test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Per-repository breakdowns are provided (Fig. 9), per-issue-length analysis (Fig. 5), successful vs non-successful coverage breakdown (Table 3), and per-model comparisons (Table 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses common failure modes: 'Adding passing tests that do not reproduce the given issue, getting stuck in loops, failing to execute the test environment correctly and adding tests with syntax errors.' Appendix D analyzes per-repo failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are reported: AUTOCODEROVER underperforms simpler methods, LIBRO's heuristics only recover half the gap to PASS@5, providing code patches has smaller impact than providing test files (Table 5), and overlap between fix and test generation is low (Table 6)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: Code Agents outperform test-specific methods (Table 2), generated tests double SWE-Agent precision (§5.3), and LLMs 'perform surprisingly well' is supported by the range of success rates reported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are modest and supported by controlled ablations. E.g., the custom diff format's effect is isolated by comparing ZEROSHOT vs ZEROSHOTPLUS (same model, same prompts, different format). SWE-AGENT vs SWE-AGENT+ isolates the test execution instruction."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges Python limitation in §6 but the title 'Testing and Validating Real-World Bug-Fixes' and abstract claims like 'LLMs generally perform surprisingly well at generating relevant test cases' are broader than the 12 Python repos tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses data contamination as an alternative explanation (§5.4, Table 7), acknowledges that popular repos may not represent common development (§6), and considers that overlap between tasks could explain correlation (Table 6)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly defines what it measures (F→P rate, change coverage) and does not overclaim these as measures of general test quality. The metrics are well-defined in §3.3 with explicit mathematical formulations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model versions are specified in §5.1: 'gpt-4-1106-preview', 'gpt-4o-mini-2024-07-18', 'Claude 3.0 Haiku', 'Claude 3.5 Sonnet', 'Mistral Large 2', 'Mixtral 7x22b'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided in the appendix: Figures 10-16 contain complete prompt text for ZEROSHOT, ZEROSHOTPLUS, SWE-AGENT, AUTOCODEROVER, and AIDER."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "§5.1 reports: 'temperature t = 0 for all zero-shot methods and agents and at t = 0.7 for LIBRO and PASS@5', API calls limited to 20, reflection steps to 3, interaction rounds to 10. LIBRO samples 5 tests."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Each agent's scaffolding is described: SWE-AGENT's command-line tools and ACI (§4.4), AIDER's repository indexing and validation (§4.4), AUTOCODEROVER's two-stage context collection and generation (§4.4). The custom diff format is formalized in §4.2 and Appendix A."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The three-stage benchmark construction process is documented in §3.2: scraping ~90K PRs, filtering for merged PRs resolving issues with test changes, filtering for F→P tests. Further filtering of 311 problematic instances is described with specific reasons."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations and Future Work' provides substantive discussion of multiple limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: Python-only limitation, popular repos not representative of common development, limited to bug reproduction (not edge case detection), and data contamination risk from historic GitHub issues."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§6 explicitly states what is not shown: 'limited to Python', 'does not measure edge case detection or global coverage increase', 'based on popular GitHub repositories, which may not be representative.' §5.4 acknowledges contamination risk limitations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full dataset is released publicly at github.com/logic-star-ai/SWT-Bench, built on top of the publicly available SWE-Bench dataset."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "§3.2 describes the three-stage data collection from SWE-Bench: scraping PRs from 12 GitHub repos, filtering for merged PRs resolving issues, filtering for F→P tests. Table 1 characterizes the resulting dataset."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is public GitHub repositories, a standard benchmark construction approach."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: ~90K PRs → filtering for merged+issue-resolving+test-changing → filtering for F→P → 2,294 instances → removing 311 problematic instances → 1,983 SWT-BENCH instances → 276 SWT-BENCH-LITE subset."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper text provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: ETH Zurich (Department of Computer Science) and LogicStar.ai. Mark Niklas Müller has dual affiliation."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Co-author Mark Niklas Müller is affiliated with LogicStar.ai, which appears to be a company related to code agents. No funding disclosure means independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. One author is affiliated with LogicStar.ai but no declaration of financial interests is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "§5.4 states 'the Knowledge Cutoff (KC) of GPT-4 (April 2023)' and analyzes contamination risk relative to this date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§5.4 explicitly investigates contamination by comparing performance on issues created before vs after GPT-4's knowledge cutoff (Table 7), finding no statistically significant difference."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "§5.4 and §6 discuss that 'most issues in SWT-BENCH have been created before the knowledge cutoff of state-of-the-art models, posing a risk for data contamination.' Table 7 provides empirical analysis. The paper also notes all methods use the same LLM so contamination affects them equally."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G reports detailed costs: Tables 9 and 10 list USD costs for each model and method on SWT-BENCH-LITE (e.g., GPT-4 SWE-AGENT: $290.71, SWE-AGENT+: $478.21)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Tables 9-10 report total API costs. Table 11 reports execution times per instance. The paper states evaluation 'can be performed on a consumer grade machine in reasonable time.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main experiments use temperature 0 (deterministic) with single runs. Table 8 shows some variance analysis for temperature ablation (n=25) but not for the primary results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states temperature t=0 for main experiments (implying single deterministic run) and t=0.7 for LIBRO/PASS@5. Table 8 explicitly states n=25 for the temperature ablation."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Default settings are used for agents (API calls=20, reflections=3, rounds=10) with no discussion of how these defaults were chosen or whether alternatives were tried beyond the ablation in Appendix C."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides ablation studies justifying the choice of hyperparameters (number of LIBRO samples, interaction rounds, temperature). Default agent settings are used without cherry-picking."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 8+ methods without correction for multiple testing. The p<0.1% claim for SWE-AGENT+ vs LIBRO is not corrected for the number of pairwise comparisons possible."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose SWT-Bench and SWE-AGENT+ (their adaptation) which outperforms other methods. No discussion of author-evaluation bias or independent verification."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Tables 9-10 report costs for all methods. Appendix C.2 (Fig. 8b) shows performance as a function of API calls. The paper enables cost-performance comparison across methods."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "§3.1-3.3 formally define what the benchmark measures (F→P as issue reproduction), §4.1 discusses how test generation differs from code repair, and §6 acknowledges construct limitations (only bug reproduction, not edge case detection)."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Different agents use fundamentally different scaffolding (SWE-Agent ACI vs AIDER repository indexing vs AutoCodeRover two-stage), yet performance differences are attributed to the methods without isolating scaffold effects from LLM effects. Table 4 varies LLMs within SWE-Agent only."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "§5.4 directly addresses temporal leakage by comparing performance on PRs before vs after GPT-4's knowledge cutoff (Table 7). §6 proposes a rolling benchmark version as mitigation."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For instance, BM-25 retrieval over the codebase could surface code near the fix location, providing hints not available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Multiple instances come from the same repositories (e.g., sympy, django are heavily represented in Fig. 2) but non-independence between instances from the same repo is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 uses a temporal split (before vs after knowledge cutoff) as a concrete detection method, comparing performance on n=83 instances in each group."
       }
     }
   },
   "claims": [
     {
       "claim": "Code Agents designed for code repair outperform systems designed specifically for test generation, with SWE-AGENT+ achieving 18.5% success rate vs LIBRO's 14.1%.",
       "evidence": "Table 2 (§5.2) shows success rates across all methods. Statistical significance at p < 0.1% is reported for SWE-AGENT+ vs LIBRO.",
       "supported": "strong"
     },
     {
       "claim": "The custom diff format (ZEROSHOTPLUS) yields almost 3x increase in success rate over standard unified diff (ZEROSHOT), from 3.6% to 9.4%.",
       "evidence": "Table 2 shows W increases from 48.6% to 89.5% and S from 3.6% to 9.4% when switching from unified diff to custom format.",
       "supported": "strong"
     },
     {
       "claim": "Generated tests double the precision of SWE-AGENT for code repair from ~20% to 47.8%.",
       "evidence": "§5.3 reports filtering code fixes using self-generated F→P tests achieves 47.8% precision at 20% recall.",
       "supported": "moderate"
     },
     {
       "claim": "Test generation and code repair are distinct tasks — overlap between solved instances is small with no statistical evidence of correlation.",
       "evidence": "Table 6 shows low overlap (7/50 for SWE-AGENT) with p-values of 72.8% and 80.4% under independence.",
       "supported": "moderate"
     },
     {
       "claim": "An ideal ensemble of the best four methods solves 71% more samples than the best single method (87 vs 51).",
       "evidence": "Figure 6 shows the Venn diagram of solved instances across the four best methods.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SWT-Bench is a novel benchmark for LLM-based test generation from GitHub issues, containing 1,983 instances from 12 Python repositories. Code Agents (especially SWE-AGENT+) outperform specialized test generation methods like LIBRO at issue reproduction, achieving 18.5% vs 14.1% success rate. Generated tests serve as an effective filter for code fixes, doubling SWE-AGENT's precision to 47.8%. Test generation and code repair show low per-instance correlation despite similar aggregate difficulty, indicating they are complementary tasks.",
   "red_flags": [
     {
       "flag": "Self-evaluation bias",
       "detail": "The authors propose both the benchmark (SWT-Bench) and the best-performing method variant (SWE-AGENT+). Their adaptations to existing agents are evaluated on their own benchmark without independent verification."
     },
     {
       "flag": "No variance on main results",
       "detail": "All primary results (Tables 2-6) are single-run at temperature 0. While deterministic, this means no assessment of sensitivity to non-deterministic factors (API non-determinism, evaluation environment variance). Table 8 shows some variance exists even at T=0."
     },
     {
       "flag": "Concentrated repository distribution",
       "detail": "Figure 2 shows sympy and django dominate the dataset. Figure 9 reveals five repositories where test generation fails entirely, meaning aggregate metrics are heavily influenced by a few repos."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "The foundational benchmark that SWT-Bench builds upon, providing the underlying dataset of GitHub issues and code patches."
     },
     {
       "title": "SWE-agent: Agent Computer Interfaces Enable Software Engineering Language Models",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "relevance": "Key Code Agent baseline adapted for test generation, introducing the ACI concept for LLM-based code interaction."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "arxiv_id": "2404.05427",
       "relevance": "Two-stage Code Agent evaluated as a baseline for test generation, demonstrating context-gathering then patch-generation approach."
     },
     {
       "title": "Large Language Models are Few-Shot Testers: Exploring LLM-Based General Bug Reproduction",
       "authors": ["Sungmin Kang", "Juyeon Yoon", "Shin Yoo"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00194",
       "relevance": "LIBRO — state-of-the-art LLM-based test generation method used as primary baseline."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "relevance": "HumanEval/Codex benchmark paper foundational to LLM code generation evaluation."
     },
     {
       "title": "CodeT: Code Generation with Generated Tests",
       "authors": ["Bei Chen", "Fengji Zhang", "Anh Nguyen"],
       "year": 2023,
       "relevance": "Uses generated tests to validate code generation, directly related to the test-as-filter approach in SWT-Bench."
     },
     {
       "title": "ChatUniTest: A Framework for LLM-Based Test Generation",
       "authors": ["Yinghao Chen", "Zehao Hu", "Chen Zhi"],
       "year": 2023,
       "relevance": "LLM-based test generation framework relevant to the automated testing survey scope."
     },
     {
       "title": "Automated Unit Test Improvement Using Large Language Models at Meta",
       "authors": ["Nadia Alshahwan", "Jubin Chheda"],
       "year": 2024,
       "arxiv_id": "2402.09171",
       "relevance": "Industry-scale LLM test generation at Meta, demonstrating practical application of automated test improvement."
     },
     {
       "title": "MAGIS: LLM-Based Multi-Agent Framework for GitHub Issue Resolution",
       "authors": ["Wei Tao", "Yucheng Zhou", "Wenqiang Zhang", "Yu Cheng"],
       "year": 2024,
       "arxiv_id": "2403.17927",
       "relevance": "Multi-agent framework for code repair, part of the Code Agent landscape being evaluated."
     },
     {
       "title": "RepairAgent: An Autonomous, LLM-Based Agent for Program Repair",
       "authors": ["Islem Bouzenia", "Premkumar T. Devanbu", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "relevance": "Autonomous LLM agent for program repair, relevant to the agentic code generation survey scope."
     },
     {
       "title": "An Empirical Evaluation of Using Large Language Models for Automated Unit Test Generation",
       "authors": ["Max Schäfer", "Sarah Nadi", "Aryaz Eghbali", "Frank Tip"],
       "year": 2024,
       "doi": "10.1109/TSE.2023.3334955",
       "relevance": "Empirical evaluation of LLMs for test generation, directly relevant baseline work."
     },
     {
       "title": "Pynguin: Automated Unit Test Generation for Python",
       "authors": ["Stephan Lukasczyk", "Gordon Fraser"],
       "year": 2022,
       "doi": "10.1145/3510454.3516829",
       "relevance": "Symbolic execution-based Python test generation tool, representing non-LLM approaches to test generation."
     }
   ]
 }