ai-research-survey

scan.json (26581B)

---

 {
   "paper": {
     "title": "The SWE-Bench Illusion: When State-of-the-Art LLMs Remember Instead of Reason",
     "authors": ["Shanchao Liang", "Spandan Garg", "Roshanak Zilouchian Moghaddam"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2506.12286"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Models achieve up to 76% accuracy identifying buggy file paths from SWE-Bench Verified issue descriptions alone without repository access, but only up to 53% on outside-repo tasks, suggesting memorization rather than reasoning. Function reproduction shows up to 35% consecutive 5-gram overlap on SWE-Bench Verified vs. only 18% on external benchmarks. Prefix completion reveals 12-32% of SWE-Bench Verified instances produce verbatim matches across models, with Claude 4 Opus reaching 31.6%.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or supplementary materials link is provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "SWE-Bench Verified and Full are public, but the authors' custom benchmarks (SWE-Bench C# described as 'internal benchmark', SWE-Bench Extra, Outside-Repo Tasks) are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or setup instructions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are included. The prompt templates are shown but no runnable pipeline is provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Figures 8-11 and Tables 2-3 are point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims performance differences across benchmarks (e.g., 76% vs 53%) without any statistical tests. Differences are asserted by visual comparison of bar charts."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage accuracies and 5-gram overlap ratios are reported with baselines, enabling readers to compute effect sizes (e.g., 76% on Verified vs. 53% on Outside-Repo is a 23 percentage point gap)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Sample sizes vary across benchmarks (500, 200, 217, 245, 39, 75) with no justification for why these sizes were chosen. The 39-task RefactorBench subset is particularly small with no explanation for why only 39 of 100 tasks were used."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run with 'default sampling settings.' No variance, standard deviation, or spread measures are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The cross-benchmark comparison design uses external benchmarks (RefactorBench, Outside-Repo Tasks, SWE-Bench C#) as implicit baselines against SWE-Bench Verified performance."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Ten contemporary models evaluated including Claude 4 Opus/Sonnet (May 2025), GPT-4.1, o3, o4-mini — all state-of-the-art at time of writing."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper presents three independent diagnostic tasks, not a single system with ablatable components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics used: accuracy, filtered accuracy, 5-gram consecutive overlap, and instance-level verbatim match percentage (Sections 3.3, Table 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs. All evaluation is automated (exact match, n-gram overlap). Manual inspection of whether identified 'memorization' cases truly reflect memorization vs. legitimate reasoning could strengthen the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The cross-benchmark design uses SWE-Bench Extra (post-cutoff issues) and Outside-Repo Tasks as held-out comparisons not subject to the memorization hypothesis."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by benchmark (6 sets), by model (10 models), and by filtered/unfiltered instances (Figures 8-11, Table 2)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative analysis of individual cases where models succeeded or failed. No examples of what memorized vs. reasoned responses look like."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 reports that on SWE-Bench Extra, most models produce negative Δ5 (closer to buggy than fixed code), supporting the claim that freshness removes memorization advantage."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 76% file-path accuracy (Figure 8, o3), 53% on outside repos (Figure 8), and 35%/18% 5-gram overlap (Figure 10) are all supported by the experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims performance gaps are 'driven by memorization' (abstract) and that models 'remember instead of reason' (title), but this is inferred from observational performance differences across benchmarks. Confounds like task difficulty, description length (Table 1 shows RefactorBench has 14.6 token descriptions vs. 451+ for others), and language differences are acknowledged but not controlled."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are specifically about SWE-Bench and the tested models. The title refers to 'State-of-the-Art LLMs' which matches the 10 models tested. Claims focus on SWE-Bench ecosystem rather than all coding benchmarks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper briefly acknowledges confounds (Table 1 shows different repo sizes and description lengths) but does not substantively discuss alternative explanations such as: task difficulty differences, structural differences between bug-fix and refactoring tasks, language-specific performance gaps for C#, or whether repo familiarity (vs. memorization) explains results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses file-path identification accuracy and n-gram overlap as proxies for 'memorization' but does not discuss the gap between these measurements and the conclusion. High file-path accuracy could reflect general architectural knowledge of popular repos rather than instance-specific memorization. High n-gram overlap could reflect common coding patterns rather than verbatim recall."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 specifies exact model versions with snapshot dates: 'gpt-4o-2024-08', 'gpt-4o-2024-05', 'gpt-4.1-2025-04-14', 'o3-2025-04-16', 'Claude 3.5 Sonnet (June 20, 2024)', 'Claude 4.0 Sonnet...May 22, 2025', etc."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt implementations are shown in Figures 2, 6, and 7, including the complete code for prompt construction. The function reproduction prompt template is in Figure 7."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.4 states max generation length (2048/4096 tokens) and 'default sampling settings' but does not specify temperature, top-p, or other sampling parameters. 'Default settings' is insufficient since defaults differ across providers and may change."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are queried directly via API with single-turn prompts."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1-3.2 describe how each benchmark was constructed: SWE-Bench C# follows SWE-Bench pipeline (75 tasks, 11 repos), Outside-Repo Tasks collected from 7 named repos (245 instances), SWE-Bench Extra from recent issues (217 tasks). Section 2 describes how diagnostic tasks are derived from benchmark instances."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists in the paper. The conclusion mentions the need for better benchmarks but does not discuss limitations of the study itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address whether its own methodology could produce false positives for memorization detection."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries stated. The paper does not clarify what its evidence does NOT show (e.g., it cannot quantify how much of SWE-Bench performance is memorization vs. reasoning)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (model outputs, per-instance results) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1-3.2 describe data collection: SWE-Bench Extra from recent issues in SWE-Bench repos post-cutoff, Outside-Repo Tasks from 7 named popular repos, SWE-Bench C# from 11 C# repos following SWE-Bench pipeline."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard and custom benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes task construction at a high level but omits pipeline details: no filtering counts for SWE-Bench Extra (how many issues were collected vs. kept), no criteria for the 39-task RefactorBench subset selection, no description of quality checks on Outside-Repo Tasks."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment appears in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Purdue University and Microsoft."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Two of three authors are Microsoft employees. Microsoft competes with OpenAI and Anthropic in AI. The paper evaluates products from these competitors for memorization/contamination, creating a potential conflict that is not disclosed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the 10 models evaluated, despite this being central to the memorization argument."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "This is the paper's central topic. Section 1 discusses how GitHub repos in training corpora overlap with SWE-Bench. The cross-benchmark design is specifically intended to detect this overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about detecting and demonstrating benchmark contamination in SWE-Bench. The three diagnostic tasks are specifically designed to measure contamination effects."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs reported despite evaluating 10 models across 6 benchmarks on 3 tasks. Section 4.3 mentions 'limited resources' as reason for not running o3 on function reproduction but gives no cost figures."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget or API costs stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are from single runs with 'default sampling settings' (Section 3.4). No seed sensitivity analysis despite using stochastic models."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Number of runs per experiment is not stated. Results appear to be single-run."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter tuning was performed; default API settings were used."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection — default settings used throughout."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes dozens of implicit comparisons across 10 models × 6 benchmarks without any statistical tests, let alone correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates existing third-party models rather than proposing its own system."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "The paper compares models on diagnostic tasks, not competing methods with different compute requirements."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about the construct validity of SWE-Bench — whether it measures reasoning ability or memorization. This is the core contribution."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding involved. Models are queried directly via single-turn API calls."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "SWE-Bench Extra (Section 3.2.2) uses recent issues from repos 'mostly created after the original SWE-Bench dataset cutoff date' as a temporal control."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The diagnostic tasks deliberately remove context (repo structure, function signatures) to test whether models succeed via memorization vs. reasoning. Filtered accuracy metric (Section 3.3) additionally removes instances where file paths appear in issue descriptions."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The cross-benchmark design explicitly compares same-repo (SWE-Bench variants) vs. different-repo (Outside-Repo Tasks) performance to test whether train/test dependence explains results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Three concrete detection methods applied: cross-benchmark performance analysis, 5-gram consecutive overlap measurement, and prefix completion verbatim matching."
       }
     }
   },
   "claims": [
     {
       "claim": "State-of-the-art models achieve up to 76% accuracy in identifying buggy file paths using only issue descriptions without repository access on SWE-Bench Verified.",
       "evidence": "Figure 8 shows OpenAI o3 achieves 76% on SWE-Bench Verified. Performance drops across benchmarks: Full-SWE-Bench (69%), SWE-Bench Extra (59%), Outside-Repo Tasks (55%).",
       "supported": "strong"
     },
     {
       "claim": "Performance on outside-repo tasks is substantially lower (up to 53%) than on SWE-Bench Verified, indicating repository-bias memorization.",
       "evidence": "Figure 8 shows all 10 models perform worse on Outside-Repo Tasks than SWE-Bench Verified, with drops of 7-21 percentage points. However, confounds exist: task construction differs, and the paper acknowledges varying repo sizes (Table 1).",
       "supported": "moderate"
     },
     {
       "claim": "Function reproduction shows much higher verbatim similarity on SWE-Bench Verified (up to 35% 5-gram overlap) than external benchmarks (up to 18%).",
       "evidence": "Figure 10 shows Claude 4 Opus achieves 34.9% on SWE-Bench Verified vs. 13.9% on Outside-Repo Tasks and 18.1% on RefactorBench.",
       "supported": "strong"
     },
     {
       "claim": "Up to 31.6% of SWE-Bench Verified instances show verbatim matches in prefix completion, indicating direct memorization.",
       "evidence": "Table 3 shows Claude 4 Opus at 31.6% instance-level verbatim match. However, this is only tested on SWE-Bench Verified with no cross-benchmark control for this particular task.",
       "supported": "moderate"
     },
     {
       "claim": "Memorization patterns are consistent across model families and vendors, indicating systematic training data exposure.",
       "evidence": "Section 4.1.3 shows all models follow the same performance hierarchy across benchmarks regardless of vendor (Figure 8).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Internal proprietary benchmark",
       "detail": "SWE-Bench C# is described as 'an internal benchmark' (Section 3.1.2) and is not publicly available, making these results unverifiable by other researchers."
     },
     {
       "flag": "No statistical tests for core claims",
       "detail": "All memorization claims rest on comparing point estimates across benchmarks (e.g., 76% vs 53%) without any significance tests. With stochastic models and varying sample sizes (39-500), some differences may not be statistically significant."
     },
     {
       "flag": "Uncontrolled confounds",
       "detail": "RefactorBench has 14.6 token average issue descriptions vs. 451+ for SWE-Bench (Table 1), making performance drops partly attributable to information availability rather than memorization. SWE-Bench C# involves a different programming language. These confounds weaken the memorization interpretation."
     },
     {
       "flag": "Competitor evaluation by Microsoft authors",
       "detail": "Two of three authors are Microsoft employees. The paper demonstrates flaws in benchmarks used to evaluate competing AI products (OpenAI, Anthropic). This potential conflict of interest is not disclosed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of its own methodological limitations, threats to validity, or what its evidence does not show."
     },
     {
       "flag": "Single-run results with stochastic models",
       "detail": "All experiments use 'default sampling settings' with no repeated runs. LLM outputs are stochastic; results could vary across runs, especially for the smaller benchmarks."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "The primary benchmark under investigation for memorization/contamination effects."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval and Codex; foundational code generation benchmark also subject to contamination concerns."
     },
     {
       "title": "Top Leaderboard Ranking = Top Coding Proficiency, Always? EvoEval: Evolving Coding Benchmarks via LLM",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Task mutation approach to contamination detection in coding benchmarks."
     },
     {
       "title": "Are Large Language Models Memorizing Bug Benchmarks?",
       "authors": ["Daniel Ramos", "Claudia Mamede", "Kush Jain", "Paulo Canelas", "Catarina Gamboa", "Claire Le Goues"],
       "year": 2025,
       "arxiv_id": "2411.13323",
       "relevance": "Directly related work on memorization detection in coding benchmarks using metric-based probing."
     },
     {
       "title": "RefactorBench: Evaluating Stateful Reasoning in Language Agents Through Code",
       "authors": ["Dhruv Gautam", "Spandan Garg", "Jinu Jang", "Neel Sundaresan", "Roshanak Zilouchian Moghaddam"],
       "year": 2025,
       "arxiv_id": "2503.07832",
       "relevance": "Alternative coding benchmark used as cross-benchmark comparison for memorization detection."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "relevance": "Major code generation benchmark relevant to evaluating LLM coding capabilities."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Major SWE-Bench agent system whose performance gains may be partially attributed to memorization."
     },
     {
       "title": "Don't Make Your LLM an Evaluation Benchmark Cheater",
       "authors": ["Kun Zhou"],
       "year": 2023,
       "arxiv_id": "2311.01964",
       "relevance": "Foundational work on benchmark contamination and data leakage in LLM evaluation."
     },
     {
       "title": "TaskEval: Assessing Difficulty of Code Generation Tasks for Large Language Models",
       "authors": ["Florian Tambon", "Amin Nikanjam", "Cyrine Zid", "Foutse Khomh", "Giuliano Antoniol"],
       "year": 2025,
       "arxiv_id": "2407.21227",
       "relevance": "Measures prompt sensitivity as contamination signal in code generation benchmarks."
     },
     {
       "title": "Memorize or Generalize? Evaluating LLM Code Generation with Evolved Questions",
       "authors": ["Wentao Chen", "Lizhe Zhang", "Li Zhong", "Letian Peng", "Zilong Wang", "Jingbo Shang"],
       "year": 2025,
       "arxiv_id": "2503.02296",
       "relevance": "AST-level mutation approach to detecting memorization vs. generalization in code generation."
     },
     {
       "title": "OpenDevin: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "Open-source SWE-Bench agent platform whose performance may be affected by memorization."
     },
     {
       "title": "Training Software Engineering Agents and Verifiers with SWE-Gym",
       "authors": ["Jiayi Pan", "Xingyao Wang", "Graham Neubig"],
       "year": 2024,
       "arxiv_id": "2412.21139",
       "relevance": "Training framework for SWE agents, relevant to understanding how training on SWE-Bench-like data affects performance."
     }
   ]
 }