ai-research-survey

scan.json (26135B)

---

 {
   "paper": {
     "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
     "authors": [
       "Carlos E. Jimenez",
       "John Yang",
       "Alexander Wettig",
       "Shunyu Yao",
       "Kexin Pei",
       "Ofir Press",
       "Karthik Narasimhan"
     ],
     "year": 2023,
     "venue": "ICLR 2024",
     "arxiv_id": "2310.06770"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SWE-bench introduces a benchmark of 2,294 real GitHub issues across 12 Python repositories for evaluating LMs on software engineering tasks. The best model (Claude 2) resolves only 1.96% of issues with BM25 retrieval, and 4.8% with oracle retrieval. Performance degrades with longer contexts, and models generate simpler, shorter edits than gold solutions. Fine-tuned SWE-Llama models are competitive with proprietary models in oracle settings but degrade significantly with BM25 retrieval due to context distribution shift.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Data, code, and leaderboard at swebench.com' and the reproducibility statement mentions uploading source code and planning open-source release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper releases the 2,294 SWE-bench task instances, the 19,000-instance training set (SWE-bench-train), and SWE-Llama model weights. 'We include the full set of 2294 SWE-bench task instances' (Reproducibility Statement)."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes executable contexts created per release version with Python versions, dependencies, and installation commands (Appendix A.3). Conda environments are specified per repository version."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The reproducibility statement (Section 9) describes organized source code with inline documentation, and separate directories for dataset collection, evaluation, inference, and training. The evaluation pipeline is described in detail in Appendix A.4."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., '1.96% of issues') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Model comparisons are made by comparing raw percentages without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance is reported as percentage resolved with baseline context (e.g., 'Claude 2 from 4.8% to 5.9%' with oracle-collapsed), providing enough context to judge magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark size of 2,294 instances is not explicitly justified. No power analysis or justification for why this number is sufficient. GPT-4 was evaluated on a 25% random subset due to budget constraints without justifying adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Single-run results only. The paper uses greedy decoding (Appendix D.2) and reports single-run numbers with no variance across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple models compared (ChatGPT-3.5, GPT-4, Claude 2, SWE-Llama 7b/13b) across multiple retrieval settings (BM25, oracle, oracle-collapsed)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Claude 2, GPT-4, and ChatGPT-3.5 were state-of-the-art models at submission time (2023). CodeLlama was the best open model for long contexts."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates retrieval method (BM25 vs oracle vs oracle-collapsed), context length, and output format (patch vs whole file generation). Table 6 shows oracle-collapsed results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Reports both % Resolved and % Apply (patch application rate). Appendix C.5 provides detailed F2P/P2P analysis with six outcome categories."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 and Appendix F provide qualitative analysis of 11 generations from SWE-Llama and Claude 2 with in-depth manual inspection of patches."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training data for SWE-Llama uses 37 repositories disjoint from the 12 evaluation repositories. A development set of 225 instances from 6 separate repositories is also provided (Appendix A.6)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 and Table 19 show per-repository breakdowns. Table 7/21 show temporal breakdowns. Table 22/23 provide detailed outcome categorization."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive failure analysis in Section 5.1 and Appendix F with 11 qualitative case studies. Discussion of models writing primitive code, greedy problem-solving, and struggling with multi-file changes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that generating whole files performs worse than patches (2.2% vs 4.8% for Claude 2), SWE-Llama degrades with BM25 retrieval, and longer contexts hurt performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are well-supported: 2,294 instances from 12 repos (Table 10), Claude 2 at 1.96% (Table 5), and claims about difficulty are backed by results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is cautious with causal language. Claims like 'performance drops considerably' with longer context are supported by controlled comparisons (same model, varying context). Ablation of retrieval method and output format use controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title asks 'Can Language Models Resolve Real-World GitHub Issues?' but the benchmark is Python-only from 12 repositories. The paper acknowledges this in Section 7 ('SWE-bench task instances are all in Python') but the framing is broader than the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses multiple factors: context length effects, retrieval quality, distribution shift for fine-tuned models, image content in issues (Section 5), and the role of code style vs. functional correctness in qualitative analyses."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's metric (% of issues resolved via test-passing) is clearly defined and the paper discusses its limitations: 'relying solely on this method is insufficient to guarantee reliable performance' and notes that LM solutions are 'less comprehensive, efficient, or readable compared to human-written solutions' (Section 7)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions stated: 'gpt-3.5-turbo-16k-0613', 'gpt-4-32k-0613', 'Claude 2', and 'CodeLlama-Python 7b/13b' (Section 4.3)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt template provided in Appendix D.3 with complete text including instructions, issue formatting, and example patch format."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Greedy decoding stated for all models (Appendix D.2). SWE-Llama training uses LoRA r=16, α=16, dropout=0.05, learning rate 6e-4, batch size 32, max 4 epochs (Appendix B.1)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are given input and generate a single patch output with no tools, feedback loops, or multi-step interaction."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The three-stage pipeline is described in detail (Section 2.1, Appendix A.1-A.3): repo selection, attribute filtering, execution-based filtering, with counts at each stage (Table 10: 93,139 PRs → 11,407 candidates → 2,294 final)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Discussion' contains 'Limitations and future directions' paragraph discussing Python-only scope, baseline simplicity, and evaluation limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific limitations discussed: Python-only (Section 7), reliance on test-passing as sole evaluation metric, popular-repository bias, models' inability to process images in issues (Section 5), GPT-4 evaluated on 25% subset due to budget (Table 5)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states: benchmark is Python-only, experiments establish baselines of 'simplest and most straight-forward approaches', and test-based evaluation is 'insufficient to guarantee reliable performance.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Full task instances released as JSON, repository mirrors created for reproducibility, and ground truth test results cached. 'We save all finalized task instances to a single .json file that is open sourced' (Appendix A.3)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Detailed three-stage collection pipeline in Section 2.1 and Appendix A.1-A.3: source repository selection from top 5,000 PyPI packages, PR scraping via GitHub API, attribute and execution-based filtering."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sourced from public GitHub repositories using a standard public benchmark construction methodology."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 10 shows exact counts at each pipeline stage for all 12 repositories: 93,139 total PRs → 11,407 post-conversion → 2,294 post-validation. Appendix A.2-A.3 detail each transformation step."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 Acknowledgements: 'We acknowledge support from the National Science Foundation under Grant No. 2239363 and an Oracle Collaborative Research award.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors list Princeton University and Princeton Language and Intelligence, except Kexin Pei at University of Chicago. No conflict with evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF and Oracle Collaborative Research award have no direct financial interest in the performance of Claude 2, GPT-4, or other evaluated models."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit training data cutoff dates stated for Claude 2, GPT-4, or ChatGPT-3.5. The paper does not specify when these models' training data ends."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 analyzes performance on instances before vs. after 2023, finding 'little difference in performance' and concluding models are 'unlikely to cheat to address issues simply by generating a more recent version of the repository.' SWE-Llama training uses disjoint repositories."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses contamination: temporal analysis in Table 7/21, disjoint repo splits for SWE-Llama, and the benchmark's 'continually updatable' nature allows evaluation on post-training instances. The paper argues temporal analysis shows models don't benefit from memorization."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Ethics statement (Section 8) confirms 'Our contributions do not involve any human subject participation.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs or per-instance inference costs reported. GPT-4 was limited to a 25% subset 'due to budget constraints' but the actual cost is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "SWE-Llama 7b training: 20 hours on 4 NVIDIA A100s. SWE-Llama 13b: 47 hours on 8 NVIDIA A100s (Appendix B.1). However, inference compute for API models is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Greedy decoding used (single deterministic run). No seed sensitivity analysis performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states greedy decoding with a single generation per instance (Appendix D.2: 'we only generate a single patch file per instance')."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "SWE-Llama training uses a specific LoRA configuration but no search budget is reported. Prompt experiments are mentioned ('slightly more or fewer lines of instructions or examples seemed to not affect overall performance substantially') but not quantified."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "SWE-Llama checkpoint selected based on validation loss on 100 held-out instances (Appendix B.1). For BM25, best context length is selected transparently (Table 2 shows all three)."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors introduce the benchmark and evaluate models on it. While they don't implement baselines of others' systems (they use API models directly), they don't discuss potential biases in benchmark design favoring certain model types."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models have vastly different context windows and compute costs. GPT-4 was evaluated on only 25% of instances. These compute differences are not systematically compared to performance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 extensively discusses what SWE-bench measures vs. prior benchmarks. Section 7 acknowledges that test-passing alone is 'insufficient to guarantee reliable performance.' Appendix C.7 explores software engineering metrics as complementary evaluation."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All models receive the same retrieval-based input format without agentic components."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 and Table 21 analyze performance by temporal partition (before/after 2023, and by year). The paper discusses that models may have seen older codebases in training and finds 'little difference in performance before or after this date.'"
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The oracle retrieval setting is explicitly acknowledged as unrealistic. The paper discusses how issue text may contain hints, and the hints_text field separates PR comments created before the initial commit to avoid solution leakage (Appendix A.2)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether task instances from the same repository are independent. Multiple instances from the same codebase may share structural similarities. Django contributes 850/2,294 instances (37%), creating strong non-independence."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Temporal analysis (Table 7) serves as an empirical leakage detection method. SWE-Llama uses disjoint repository splits. The continually-updatable design allows post-training-cutoff evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "Claude 2 resolves only 1.96% of SWE-bench issues with BM25 retrieval, making it the best-performing model in this setting.",
       "evidence": "Table 5 shows Claude 2 at 1.96% resolved with BM25, compared to ChatGPT-3.5 (0.17%), GPT-4-turbo (1.31%), and SWE-Llama 13b (0.70%).",
       "supported": "strong"
     },
     {
       "claim": "Performance degrades as total context length increases.",
       "evidence": "Figure 5 shows Claude 2 performance drops with longer inputs. Table 2 shows performance decreasing from 1.96% (13k) to 1.22% (50k) for Claude 2.",
       "supported": "strong"
     },
     {
       "claim": "Oracle retrieval improves performance: Claude 2 achieves 4.80% with oracle files.",
       "evidence": "Table 18 shows Claude 2 at 4.80% with oracle retrieval vs. 1.96% with BM25 (Table 5).",
       "supported": "strong"
     },
     {
       "claim": "SWE-Llama models are competitive with Claude 2 in oracle settings but degrade significantly with BM25 retrieval.",
       "evidence": "Table 18 shows SWE-Llama 13b at 3.97% vs Claude 2 at 4.80% (oracle). Table 5 shows SWE-Llama 13b at 0.70% vs Claude 2 at 1.96% (BM25). Section 5 attributes this to context distribution shift.",
       "supported": "strong"
     },
     {
       "claim": "Model performance does not correlate with issue resolution date, suggesting models are not 'cheating' by memorizing solutions.",
       "evidence": "Table 7 shows similar performance before and after 2023 for most models, with extended analysis in Table 21 across 6 temporal partitions.",
       "supported": "moderate"
     },
     {
       "claim": "Models generate shorter, simpler edits than gold patches.",
       "evidence": "Table 8 shows model patches average 30.1 total lines vs 74.5 for gold patches, edit fewer files (1.0 vs 1.7), and fewer functions.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "GPT-4 evaluated on 25% subset only",
       "detail": "Due to budget constraints, GPT-4 was evaluated on only 574 of 2,294 instances in BM25 27k and oracle settings. This may affect reliability of GPT-4 comparisons, though Table 20 suggests the subset is representative."
     },
     {
       "flag": "No error bars or variance measures",
       "detail": "All results are single-run with greedy decoding. While greedy decoding is deterministic, no uncertainty quantification is provided for the aggregate metrics over the dataset."
     },
     {
       "flag": "Heavy repository imbalance",
       "detail": "Django contributes 850/2,294 (37%) of all task instances, potentially biasing aggregate results toward Django-style issues. Performance varies significantly across repos (Figure 4)."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "relevance": "HumanEval benchmark — the dominant code generation benchmark that SWE-bench is designed to move beyond."
     },
     {
       "title": "Beyond the imitation game: Quantifying and extrapolating the capabilities of language models",
       "authors": ["Aarohi Srivastava"],
       "year": 2023,
       "relevance": "BIG-bench — large-scale LM evaluation framework that SWE-bench positions against as a more focused benchmark."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Rozière"],
       "year": 2023,
       "relevance": "Base model for SWE-Llama fine-tuning; key open-source code model."
     },
     {
       "title": "AgentBench: Evaluating LLMs as agents",
       "authors": ["Xiao Liu"],
       "year": 2023,
       "relevance": "Agent evaluation benchmark that SWE-bench relates to for agent-based code editing."
     },
     {
       "title": "An analysis of the automatic bug fixing performance of chatgpt",
       "authors": ["Dominik Sobania"],
       "year": 2023,
       "relevance": "LLM-based automated program repair evaluation, directly related to SWE-bench's task."
     },
     {
       "title": "Conversational automated program repair",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Conversational approach to LLM-based program repair — related methodology to iterative SWE-bench solving."
     },
     {
       "title": "Lost in the middle: How language models use long contexts",
       "authors": ["Nelson F. Liu"],
       "year": 2023,
       "arxiv_id": "2307.03172",
       "relevance": "Context utilization study that corroborates SWE-bench finding that models struggle with long-context code localization."
     },
     {
       "title": "Holistic evaluation of language models",
       "authors": ["Percy Liang"],
       "year": 2022,
       "relevance": "HELM benchmark framework — multi-task LM evaluation that SWE-bench positions as alternative to potpourri-style evaluation."
     },
     {
       "title": "AI safety subproblems for software engineering researchers",
       "authors": ["David Gros"],
       "year": 2023,
       "relevance": "Discusses safety implications of AI-driven software engineering, referenced in SWE-bench societal impact discussion."
     },
     {
       "title": "MultiPL-E: A scalable and extensible approach to benchmarking neural code generation",
       "authors": ["Federico Cassano"],
       "year": 2022,
       "relevance": "Multi-language code generation benchmark that extends HumanEval; SWE-bench positions against as more realistic."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation",
       "authors": ["Xueying Du"],
       "year": 2023,
       "relevance": "Class-level code generation benchmark representing more complex coding tasks than HumanEval."
     },
     {
       "title": "WebArena: A realistic web environment for building autonomous agents",
       "authors": ["Shuyan Zhou"],
       "year": 2023,
       "relevance": "Realistic agent evaluation environment; parallel to SWE-bench's approach of grounding evaluation in real-world tasks."
     }
   ]
 }