ai-research-survey

scan.json (28142B)

---

 {
   "paper": {
     "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
     "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
     "year": 2024,
     "venue": "NeurIPS 2024",
     "arxiv_id": "2405.15793"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Data, code, and leaderboard at swe-agent.com' and references open-source GitHub repositories. Section E.2 explicitly states 'we open source all of our resources.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "They use the publicly available SWE-bench dataset and HumanEvalFix benchmark. Section E.2 states 'We also open source all inference and evaluation artifacts (e.g., trajectories, code generations, evaluation execution traces, analysis notebooks).'"
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section A.2 describes the Docker-based environment setup. The system uses Docker containers for reproducible execution, and the configuration system is documented in Section A.3."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section E.2 states 'Practitioners should be able to easily recover our findings by running the agent with simple scripts. We provide extensive text and video documentation describing how to run and modify different parts of the codebase.'"
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 10 reports mean ± std across 6 runs: '17.94±0.49'. Figure 4 shows pass@k performance across 6 runs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Comparisons between SWE-agent and baselines (e.g., Shell-only 11.0% vs SWE-agent 18.0%) are made without any formal statistical testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes in context: '10.7 percentage points more instances than the baseline agent' (Section 1), '64% relative increase compared to Shell-only' (Section 5), and detailed percentage improvements in Table 3 ablations."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 300 instances (SWE-bench Lite) is sufficient for ablations, or why 37 instances were used for the hyperparameter sweep. The dev set of 37 instances is particularly small."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 10 reports variance across 6 runs with std dev (17.94±0.49). Figure 4 shows pass@k across 6 runs. The paper notes 'average performance variance is relatively low, but per-instance resolution can change considerably.'"
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares against RAG baselines (from Jimenez et al.) and a Shell-only interactive baseline adapted from Yang et al. (InterCode)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The RAG baseline is from the original SWE-bench paper (2024), and the Shell-only baseline uses the same underlying models (GPT-4 Turbo, Claude 3 Opus). These were contemporary at submission time."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides detailed ablations of individual ACI components: editor (with/without linting, no edit), search (summarized, iterative, no search), file viewer window size (30, 100, full), and context management (last 5 obs, full history, without demo)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports % Resolved (pass@1), $ Avg. Cost, pass@k for k∈{1-6}, and file localization F1 score. Table 2 also reports Pass@1 across three languages for HumanEvalFix."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The failure mode analysis uses both automated LM categorization and a hand-labeled validation set of 15 instances, with 87% agreement reported (Section 5.2, Figure 8). The qualitative analysis in Section D provides detailed manual trajectory walkthroughs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports main results on SWE-bench test set (2,294 instances) and SWE-bench Lite test set (300 instances), separate from the dev split used for design decisions. Section B.1 uses a 37-instance dev split for hyperparameter sweeps."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 provides per-repository performance breakdown across 12 repositories. Table 7 provides temporal breakdown by year. Figure 8 provides failure mode distribution. Table 8 provides per-turn action pattern analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 and Figure 8 provide a detailed failure mode taxonomy (9 categories) with distribution analysis. Section D provides two full qualitative failure case studies. Section B.3.3 discusses editing failure patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that iterative search performed worse than no search at all (12.0% vs 15.7%, Table 3). It also reports that showing the full file hurts performance (12.7% vs 18.0%). Failed experiments with Llama 3 and DeepSeek Coder are mentioned in Section 4."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 12.5% on SWE-bench and 87.7% on HumanEvalFix are supported by Tables 1 and 2. The claim of 'state-of-the-art' and 'far exceeding the previous state-of-the-art' is supported by the comparison to prior 3.8% rate."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claim that ACI design improves performance is supported by controlled ablation studies (Table 3) where individual components are removed/modified while holding the LM fixed. The study design (fixed LM, varying interface) is explicitly stated in Section 2."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper introduces the concept of ACI as a general framework but tests only on Python tasks (SWE-bench) with only two LMs (GPT-4 Turbo, Claude 3 Opus). The title claims 'Automated Software Engineering' broadly. Section E.3 acknowledges scope is 'exclusively focused on programmatic tasks' but the framing throughout suggests broader generality."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why the ACI improves performance. For example, it does not consider whether the improvement is primarily from the demonstrations/prompts rather than the interface design, or whether the ACI simply provides more tokens of context about the task."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly defines % Resolved as 'the proportion of instances for which all tests pass successfully after the model generated patch is applied' (Section 4). It does not overclaim this as general SE capability — it frames results specifically in terms of SWE-bench task resolution."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 specifies exact model versions: 'gpt-4-1106-preview' and 'claude-3-opus-20240229'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full prompt templates are provided in Appendix C (Figures 27-32), including the system prompt, instance template, demonstration template, next step templates, and error message template. Section A.3 describes the configuration system."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports the hyperparameter sweep results including temperature (0.0, 0.2), window size (100, 200), and history processing method. The final configuration uses temperature 0.0, window 100, last 5 observations."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The ACI scaffolding is described in extensive detail across Sections 2-3 and Appendix A. All custom commands are documented in Table 4. The file viewer, editor, search system, and context management are described with implementation details."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the datasets, their splits, and how they are used. For HumanEvalFix, Section B.7 describes how the dataset was adapted for the SWE-agent setting. SWE-bench's data collection procedure is referenced from Jimenez et al."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section E.3 is titled 'Limitations & Future Work' and discusses specific limitations. Section 7 (Discussion) also acknowledges scope constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section E.3 discusses specific limitations: the small toolkit, manual ACI development process, exclusive focus on Python/programmatic tasks. Section E.1 discusses specific security risks of executing LM-generated code."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section E.3 explicitly states 'the scope of SWE-agent is exclusively focused on programmatic tasks like software engineering and code generation' and discusses what was NOT tested (other domains, other programming languages beyond the benchmarks)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section E.2 states 'We also open source all inference and evaluation artifacts (e.g., trajectories, code generations, evaluation execution traces, analysis notebooks). The results presented in the main and supplementary parts of this paper can be fully rendered from the data.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data comes from established benchmarks (SWE-bench, HumanEvalFix) whose collection procedures are documented in their respective papers. The paper references these sources clearly."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses automated benchmarks (SWE-bench, HumanEvalFix) as data sources."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from benchmark task instance to evaluation is documented: task instances are loaded, Docker containers are set up, the agent interacts, patches are generated, and evaluation runs unit tests. Section A.2 describes the implementation pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section states 'We acknowledge support from an Oracle Collaborative Research award and the National Science Foundation under Grant No. 2239363.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Princeton Language and Intelligence, Princeton University. No commercial affiliations with the evaluated products (OpenAI, Anthropic)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding from Oracle and NSF. Neither has a direct financial interest in SWE-agent's performance on SWE-bench. The research is from an academic institution."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is included in the paper. While the authors are academic researchers, the absence of a formal declaration means this criterion is not satisfied."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for GPT-4 Turbo or Claude 3 Opus, even though it tests them on benchmarks that could overlap with training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section B.2 (Temporal Analysis, Table 7) analyzes whether performance correlates with issue creation year, stating 'There is no clear correlation between a task instance's creation year and its resolution rate' — this addresses potential test pollution from training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the temporal analysis in B.2 controls for 'possible test pollution,' the paper does not directly address whether SWE-bench task instances or their solutions appeared in GPT-4 Turbo's or Claude 3 Opus's training data. HumanEvalFix contamination is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports $ Avg. Cost per resolved instance for all configurations. Section 4 states a $4 per-instance budget. Table 15 shows cost distributions for resolved vs unresolved instances."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The per-instance budget of $4 is stated in Section 4. Total costs can be inferred from the per-instance costs and number of instances. The paper uses API-based models so GPU hours are not applicable."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 10 reports results across 6 separate runs with mean and standard deviation (17.94±0.49). Figure 4 shows pass@k across these 6 runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 6 runs for variance analysis (Table 10, Section B.5). Main results are reported as pass@1 from individual runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 shows the full hyperparameter sweep: 2 temperatures × 2 window sizes × 2 history settings × 2 models = 16 configurations, each averaged over 5 samples on 37 dev instances."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section B.1 describes the sweep on dev split instances, and the best configuration (temperature 0.0, window 100, last 5 observations) is selected based on dev performance before testing. All configurations are reported in Table 5."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple ablations and comparisons are reported (Table 3 has ~10 comparisons) without any correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement both SWE-agent and the Shell-only baseline. They do not discuss the bias of implementing and tuning their own system vs. baselines. The Shell-only baseline is described as 'adapted from' InterCode but may not have received the same development effort."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports both % Resolved and $ Avg. Cost together. The paper notes SWE-agent is '8-13x more costly but yields a 6.7-fold improved % Resolved rate' compared to RAG. Figure 15 shows cost vs resolution distributions."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 discusses why SWE-bench is a suitable benchmark, noting it 'unites many separate SE tasks' and uses 'rigorous, execution-based evaluation with human-written unit tests.' The paper also evaluates on HumanEvalFix to test a different aspect (code editing)."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The scaffold IS the primary variable under study. The paper explicitly holds the LM fixed and varies the ACI (Section 2: 'we assume a fixed LM and focus on designing the ACI'). Table 1 shows the same models under different scaffolds (RAG, Shell-only, SWE-agent)."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section B.2 provides a temporal analysis (Table 7) showing performance by task instance creation year, controlling for potential temporal leakage. The paper notes 'no clear correlation' between issue age and resolution rate."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the issue descriptions or test file paths in SWE-bench could leak information about the solution. The agent receives the full issue text which may contain hints."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench instances come from only 12 repositories. The paper does not discuss whether training data for GPT-4/Claude included code from these same repositories, creating non-independence between train and test data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. The temporal analysis is correlational but not a detection method for data contamination."
       }
     }
   },
   "claims": [
     {
       "claim": "SWE-agent with GPT-4 Turbo solves 12.47% of SWE-bench test instances, substantially outperforming prior non-interactive 3.8% state-of-the-art.",
       "evidence": "Table 1 shows 12.47% (286/2,294) resolved on full SWE-bench test set vs 3.79% for RAG with Claude 3 Opus.",
       "supported": "strong"
     },
     {
       "claim": "SWE-agent's ACI yields a 64% relative increase over the Shell-only baseline (18.0% vs 11.0% on SWE-bench Lite).",
       "evidence": "Table 1 and Section 5 report these numbers. The Shell-only baseline uses the same GPT-4 Turbo model.",
       "supported": "strong"
     },
     {
       "claim": "The ACI concept is portable across LMs: SWE-agent with Claude 3 Opus achieves 10.46% on SWE-bench.",
       "evidence": "Table 1 shows Claude 3 Opus achieves 10.46% on full SWE-bench, demonstrating the ACI works with a different model.",
       "supported": "moderate"
     },
     {
       "claim": "SWE-agent achieves 87.7% pass@1 on HumanEvalFix Python, far exceeding prior methods.",
       "evidence": "Table 2 shows 87.7% vs next-best 57.9% (WaveCoder-DS-6.7B). Results also reported for JS (89.7%) and Java (87.9%).",
       "supported": "strong"
     },
     {
       "claim": "Editing guardrails (linting) improve performance by 3 percentage points (18.0% vs 15.0%).",
       "evidence": "Table 3 ablation study shows edit with linting at 18.0% vs edit without linting at 15.0%.",
       "supported": "moderate"
     },
     {
       "claim": "Iterative search (UI-inspired) actually hurts performance compared to no search tools (12.0% vs 15.7%).",
       "evidence": "Table 3 shows iterative search at 12.0% vs no search at 15.7%. Section 5.1 explains agents exhaustively iterate through results.",
       "supported": "moderate"
     },
     {
       "claim": "Successful runs complete earlier and cheaper than unsuccessful ones (median $1.21/12 steps vs $2.52/21 steps).",
       "evidence": "Section 5.2, Table 15, and Figure 15 show the distribution differences.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SWE-agent introduces the agent-computer interface (ACI) concept, demonstrating that LM agents benefit from custom-designed interfaces tailored to their capabilities. With GPT-4 Turbo, SWE-agent resolves 12.47% of SWE-bench tasks, a 3.3x improvement over prior non-interactive approaches. Detailed ablations show each ACI component contributes meaningfully: the file editor alone accounts for 7.7 percentage points, and linting guardrails add 3 points. Interestingly, a UI-inspired iterative search interface actually hurts performance because agents exhaustively iterate through results rather than refining queries.",
   "red_flags": [
     {
       "flag": "Small ablation sample",
       "detail": "Ablations in Table 3 are performed on SWE-bench Lite (300 instances) with single runs, while the hyperparameter sweep uses only 37 dev instances averaged over 5 samples. Small samples make ablation differences potentially unreliable."
     },
     {
       "flag": "No significance tests for ablation claims",
       "detail": "Table 3 reports percentage differences (e.g., 3.0 point improvement from linting) without any significance testing. With 300 instances, some differences may not be statistically significant."
     },
     {
       "flag": "Self-comparison bias",
       "detail": "The authors designed and tuned SWE-agent while using their own implementation of the Shell-only baseline. The baseline may not have received comparable development effort."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig", "S. Yao", "K. Pei", "O. Press", "K. R. Narasimhan"],
       "year": 2024,
       "relevance": "Defines the SWE-bench benchmark used as primary evaluation, establishing the task formulation for real-world software engineering."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["S. Yao", "J. Zhao", "D. Yu"],
       "year": 2023,
       "relevance": "Foundational agentic reasoning framework (thought+action) used by SWE-agent's interaction loop."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["N. Shinn", "F. Cassano", "E. Berman"],
       "year": 2023,
       "relevance": "Demonstrates LM agents using execution feedback for code generation, motivating SWE-agent's interactive approach."
     },
     {
       "title": "InterCode: Standardizing and benchmarking interactive coding with execution feedback",
       "authors": ["J. Yang", "A. Prabhakar", "K. R. Narasimhan", "S. Yao"],
       "year": 2023,
       "relevance": "Provides the interactive coding framework that SWE-agent's Shell-only baseline and environment are built upon."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "relevance": "Introduces HumanEval/Codex, the foundational code generation benchmark against which many systems are measured."
     },
     {
       "title": "Automated repair of programs from large language models",
       "authors": ["Z. Fan", "X. Gao", "M. Mirchev", "A. Roychoudhury", "S. H. Tan"],
       "year": 2023,
       "relevance": "Prior work on LLM-based automated program repair, one of the SE tasks unified under SWE-bench."
     },
     {
       "title": "LLM agents can autonomously hack websites",
       "authors": ["R. Fang", "R. Bindu", "A. Gupta", "Q. Zhan", "D. Kang"],
       "year": 2024,
       "relevance": "Demonstrates LM agents applied to offensive security, showing agentic capabilities extend to adversarial domains."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["S. Hong", "M. Zhuge", "J. Chen"],
       "year": 2023,
       "relevance": "Multi-agent framework for software engineering, representing an alternative architectural approach to SWE-agent."
     },
     {
       "title": "Lost in the middle: How language models use long contexts",
       "authors": ["N. F. Liu", "K. Lin", "J. Hewitt"],
       "year": 2023,
       "relevance": "Demonstrates that distracting context harms LM performance, motivating SWE-agent's context management design."
     },
     {
       "title": "OctoPack: Instruction tuning code large language models",
       "authors": ["N. Muennighoff", "Q. Liu", "A. R. Zebaze"],
       "year": 2024,
       "relevance": "Introduces HumanEvalFix benchmark used as secondary evaluation in SWE-agent."
     },
     {
       "title": "Cognitive architectures for language agents",
       "authors": ["T. Sumers", "S. Yao", "K. Narasimhan", "T. L. Griffiths"],
       "year": 2023,
       "relevance": "Theoretical framework for understanding language agents, relevant to ACI design principles."
     },
     {
       "title": "Language Agent Tree Search unifies reasoning acting and planning in language models",
       "authors": ["A. Zhou", "K. Yan", "M. Shlapentokh-Rothman"],
       "year": 2023,
       "relevance": "Alternative agentic search strategy achieving high HumanEval performance (94.4%), showing benchmark saturation."
     }
   ]
 }