ai-research-survey

scan.json (35484B)

---

 {
   "paper": {
     "title": "SWE-rebench: An Automated Pipeline for Task Collection and Decontaminated Evaluation of Software Engineering Agents",
     "authors": [
       "Ibragim Badertdinov",
       "Alexander Golubev",
       "Maksim Nekrashevich",
       "Anton Shevtsov",
       "Simon Karasik",
       "Andrei Andriushchenko",
       "Maria Trofimova",
       "Daria Litvintseva",
       "Boris Yangel"
     ],
     "year": 2025,
     "venue": "NeurIPS 2025 Track on Datasets and Benchmarks",
     "arxiv_id": "2505.20411",
     "doi": "10.48550/arXiv.2505.20411"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SWE-rebench introduces a fully automated pipeline that extracts 21,336 verifiable SWE tasks from 3,468 GitHub repositories, eliminating the manual curation bottleneck of prior benchmarks. Comparing model performance on SWE-rebench (fresh 2025 tasks) versus SWE-bench Verified reveals substantial performance drops for several models (e.g., DeepSeek-V3-0324 drops from 39.7% to 21.3%), suggesting potential contamination effects in older benchmarks. GPT-4.1 is the only model showing notable performance decline between January and March-April 2025 task subsets. Using a standardized ReAct scaffold and 5 runs per model, the paper demonstrates that DeepSeek-V3 models are the strongest open-source performers while Qwen3's thinking mode provides no measurable advantage over no-think mode.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 states 'Accompanying code for utilizing the dataset, including scripts for tasks evaluation, is available on GitHub.' The paper also links to the HuggingFace dataset and leaderboard website."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The SWE-rebench dataset of 21,336 tasks is publicly available on HuggingFace Datasets (https://huggingface.co/datasets/nebius/SWE-rebench), as stated in Section 2.4 and footnote 2."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "While the paper describes the evaluation infrastructure (vLLM on 2 nodes with 8×H200 GPUs each, Appendix J) and each task instance includes install_config and pinned dependencies, no environment specification (requirements.txt, Dockerfile) is provided for reproducing the pipeline or evaluation infrastructure itself."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper references code on GitHub and data on HuggingFace, but does not include step-by-step reproduction instructions in the paper itself. The evaluation setup is described at a high level in Appendix J but lacks specific commands to replicate the full pipeline or benchmark runs."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 report SEM (Standard Error of the Mean) for all models across both benchmark subsets, providing uncertainty quantification for the resolved rate metric."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No formal statistical significance tests (p-values, t-tests, etc.) are reported. Claims about model differences and contamination effects are based on comparing point estimates and SEM without formal hypothesis testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 report absolute resolved rates on both SWE-bench Verified and SWE-rebench, allowing direct computation of effect magnitudes. For example, DeepSeek-V3-0324 drops from 39.7% to 21.3% across benchmarks, providing clear magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 294 tasks from 169 repositories (Appendix H describes filtering criteria), but no justification is given for why 294 tasks is sufficient for the claims made, and no power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 states 'we run each model five times on the full benchmark' and reports SEM across runs in Tables 1 and 2, along with pass@5 as an additional variance-aware metric."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 compare 13 models against each other, including GPT-4.1, DeepSeek-V3, Qwen3, LLaMa-4, and others. Table 2 additionally compares performance on SWE-bench Verified as a reference baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include the latest releases: GPT-4.1 (April 2025), DeepSeek-V3-0324 (March 2025), Qwen3 (April 2025), LLaMa-4 (2025), and gemma-3 (2025). These are contemporary state-of-the-art models."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 (Appendix C) compares installation approaches (agentless with 1/3/10 candidates vs agent-based), but this is method selection for a single pipeline component, not a systematic ablation of the pipeline stages. No ablation of the benchmark evaluation design (e.g., effect of filtering criteria, quality assessment) is provided."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: resolved rate (mean across 5 runs), SEM (standard error of the mean), and pass@5. These capture different aspects of model performance."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated via test-suite pass/fail. No human evaluation of model outputs (patches, trajectories) is included in the benchmark evaluation. The SWE-bench Verified human annotations were used only for training the quality classifier, not for evaluating model outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The 294 benchmark tasks are used exclusively for evaluation. The paper explicitly criticizes other approaches for 'scaffoldings are often developed and tuned on subsets from SWE-bench' (Section 3.1) and uses a fixed, standardized scaffold with no task-specific tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by temporal subset (January 2025 vs March-April 2025 in Table 1), by benchmark (SWE-bench Verified vs SWE-rebench in Table 2), and by number of files changed (Appendix F: 1 file 28.6%, 2 files 20.6%, ≥3 files 17.5%)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 discusses Qwen2.5-Coder-32B-Instruct's instruction-following failures ('frequently hallucinated environment responses or enters loops of formatting errors'). Appendix J.2 provides a detailed trajectory example of this failure mode. Appendix B.5 discusses pipeline failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 reports that Qwen3's think mode provides 'no measurable advantage' over no-think mode, GPT-4.1 shows performance decline on the March-April subset, and Qwen2.5-Coder-32B-Instruct 'underperforms expectations.' Pipeline limitations (31% success rate for installation recipes) are also reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: the automated pipeline is described in Section 2, the 21,000+ task dataset is documented with statistics (Table 7), and the contamination analysis is presented in Tables 1-2 with temporal comparisons showing performance differences between SWE-bench Verified and SWE-rebench."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'performance of some language models might be inflated due to contamination issues' (abstract). While hedged with 'might,' the implication is that contamination causes the performance gap between SWE-bench Verified and SWE-rebench. However, the gap could be explained by task difficulty differences, repository diversity, or distribution shift between benchmarks. The temporal comparison (Jan vs Mar-Apr) provides suggestive evidence but doesn't control for confounds."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Evaluation of Software Engineering Agents' broadly while all results are Python-only. Although the abstract mentions 'Python-based SWE tasks' and Section 4 acknowledges the Python limitation, the title and overall framing suggest generality beyond what was tested. The benchmark uses only 169 repositories."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes the performance gap between SWE-bench Verified and SWE-rebench primarily to contamination, without substantively discussing alternative explanations such as differences in task difficulty distribution, repository familiarity, or temporal recency making tasks inherently harder. The limitations section discusses automated quality and Python scope but not alternative explanations for the contamination finding."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 specifically identifies two things being measured: '(1) The ability to comprehend a real-world software issue, devise a plan, implement a correct code patch' and '(2) The ability to follow instructions and operate within a structured agentic framework.' The proxy (test-suite pass rate) and the claimed capabilities are closely aligned, and the paper discusses test patch quality assessment (Section 2.4) as a factor affecting measurement validity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 lists specific model identifiers with version dates where applicable: 'gpt-4.1-2025-04-14', 'DeepSeek-V3-0324', 'DeepSeek-V3-1226', 'Qwen3-235B-A22B', 'Qwen2.5-72B-Instruct', etc. Open-source models include size specifications, and proprietary models include snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full system prompt used for all model evaluations is provided in Appendix I, spanning several pages with complete tool descriptions, formatting instructions, and a demonstration trajectory. The paper states 'we share the exact system prompt used for all model evaluations' (Section 3.2)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Appendix J states they used 'generation hyperparameters recommended by the respective model developers (e.g., temperature, p in top-p, max context length etc.)' and 'standardize the context length to 128K tokens,' but does not list the actual parameter values used for each model. Readers cannot determine the specific temperature or top-p settings applied."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The scaffolding is described as a 'minimal ReAct-style agentic framework' (Section 3.2). The complete system prompt with all tools (create, edit, goto, open, replace, scroll, search_file, submit) is provided in Appendix I. The paper explicitly states function-calling is not used and all interaction is text-based command generation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The four-stage pipeline is documented in detail (Sections 2.1-2.4): preliminary collection with specific filtering criteria (Section 2.1), automated installation (Section 2.2), execution-based verification (Section 2.3), and quality assessment (Section 2.4). Table 6 (Appendix L) provides the data funnel with counts at each stage. Benchmark filtering criteria are listed in Appendix H."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 is titled 'Discussion and limitations' and provides substantive discussion of multiple limitations including automated quality assessment imperfections, Python-only scope, and scalability trade-offs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses specific threats: 'our LLM-based approach to generating installation instructions... was validated on a limited set of 18 repositories,' 'the automated task quality assessment... cannot fully replicate nuanced human judgment,' and 'fully automated pipeline may result in some tasks being imperfectly described or unsolvable solely from the issue.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 explicitly states: 'The initial release of SWE-rebench is limited exclusively to Python, restricting its immediate applicability to other language ecosystems.' The paper also notes that automated assessment 'can lead to lower absolute success rates compared to manually curated benchmarks' and describes what the benchmark does NOT test."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full SWE-rebench dataset with all 21,336 task instances and their annotations is publicly available on HuggingFace Datasets. Appendix E shows the complete schema with all fields. Each instance includes problem_statement, patches, test results, and metadata."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes data collection in detail: sources (GitHub Archive and GitHub), approximately 450,000 pull requests from 30,000+ repositories, with specific inclusion criteria (permissive licenses, Python >75%, merged PRs, issue descriptions >10 characters, 1-15 files changed, etc.)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The sample recruitment is well-described: repositories were selected based on permissive licenses (listed in Appendix D), Python code proportion >75%, and PR/issue linking criteria. The filtering funnel from 10M PRs to 21K tasks is documented in Table 6."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The four-stage pipeline is fully documented with counts: ~10M PRs → ~450K candidates → ~150K filtered → ~21K valid tasks (Table 6). Each filtering criterion is specified (Section 2.1), and acceptance rates are reported at each stage (5%, 33%, 14%, 100%)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. All authors are affiliated with Nebius, and the work uses Nebius infrastructure (TractoAI), but there is no formal funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All nine authors are listed with 'Nebius' affiliation in the paper header. The paper also discloses use of Nebius's TractoAI platform for distributed computing."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Nebius is a cloud computing/AI infrastructure company (owner of TractoAI) with commercial interests in the SWE agent ecosystem. The benchmark promotes use of infrastructure for training and evaluating agents, which aligns with Nebius's business interests. This potential conflict is not discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Nebius's commercial interests in the AI agent infrastructure space are not disclosed as a potential conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses model release dates as a proxy (models released after March 1, 2025 are marked with asterisks in Table 1) but does not state the actual training data cutoff dates for any of the evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "This is a central contribution of the paper. Section 3.1 discusses contamination risk for SWE-bench ('models released afterward may have been exposed to its data during training'). Section 3.2 describes their temporal decontamination approach: tasks created in 2025 after model release dates. Tables 1-2 explicitly analyze potential contamination effects."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core innovation is addressing contamination. Section 3.2 describes continuous dataset updates using fresh tasks from 2025, temporal tracking of issue creation dates against model releases, and explicit marking of potentially contaminated evaluations on the leaderboard."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses automated pipelines and model evaluations on GitHub-sourced tasks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from public GitHub repositories and automated model evaluations."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Repository and task inclusion criteria are documented separately under data_preprocessing."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or evaluators requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Appendix J reports that 'a single run over the full task set required approximately 7 hours' for DeepSeek-V3 on 2 nodes with 8×H200 GPUs. However, wall-clock time is reported for only one model, with no token counts, API costs, or per-task costs for any model."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (2 nodes × 8×H200 GPUs per model, Appendix J) and one model's runtime is given (~7 hours for DeepSeek-V3). But total GPU-hours across all 13 models × 5 runs, pipeline compute costs (TractoAI), and quality model fine-tuning costs are not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix J states 'each model was run 5 times on the full benchmark using different random seeds.' SEM is reported in Tables 1-2, capturing cross-seed variability."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 explicitly states 'we run each model five times on the full benchmark' and Appendix J confirms '5 times... using different random seeds.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 and Appendix J explicitly state that 'default generation hyperparameters as recommended by model developers' were used with no tuning, making the search budget transparently zero."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All models use their default configurations with no selection or tuning. All 13 models' results are reported in Tables 1-2 without cherry-picking configurations."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 13 models and makes multiple claims about relative performance differences without applying any correction for multiple comparisons (e.g., Bonferroni, Holm)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While Nebius doesn't evaluate its own model, the authors designed the benchmark and evaluation scaffold. Potential biases in benchmark construction (task selection, filtering criteria, scaffold design) that could inadvertently favor certain models are not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of vastly different sizes are compared (27B to 235B parameters) without analyzing or controlling for compute budget differences. Performance is not reported as a function of model size or inference compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 explicitly describes what the benchmark measures ('The ability to comprehend a real-world software issue... and implement a correct code patch' and 'The ability to follow instructions and operate within a structured agentic framework'). Section 2.4 discusses quality assessment of tasks. Limitations section acknowledges that automated curation may produce imperfect tasks."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "This is a major design principle. Section 3.2 states 'every model is assessed by using the same minimal ReAct-style agentic framework, identical prompts and default generation hyperparameters.' Section 3.1 explicitly identifies scaffolding variability as a problem in prior work. Function calling is disabled to equalize evaluations."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "This is the paper's central contribution. Section 3.2 describes using 'automated pipeline for a continuous supply of fresh tasks' with temporal tracking. Tasks are from January and March-April 2025, after most models' training periods. Models released after March 2025 are marked in Table 1."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the evaluation setup could leak answer information through the provided context. The focus is entirely on temporal/training contamination. Whether the issue descriptions, repository structure, or test names provide unintended hints toward solutions is not analyzed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses 169 repositories, but there is no discussion of whether training data and test data share repositories, code patterns, or problem structures. Whether the 294 benchmark tasks are structurally independent from data seen during model training (beyond temporal filtering) is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper applies a concrete temporal decontamination method: tasks are sourced from issues created in 2025, and 'we precisely track the creation dates of the issues and their corresponding pull requests against model release dates' (Section 3.2). Potentially contaminated evaluations are explicitly marked on the leaderboard."
       }
     }
   },
   "claims": [
     {
       "claim": "The automated pipeline produces 21,336 verifiable SWE tasks from 3,468 GitHub repositories without manual curation.",
       "evidence": "Section 2 describes the four-stage pipeline. Table 6 (Appendix L) shows the data funnel: ~10M PRs → ~450K candidates → ~150K filtered → ~21K valid tasks. Table 7 provides dataset statistics.",
       "supported": "strong"
     },
     {
       "claim": "The automated installation recipe generation successfully configures environments for 31% of all repositories.",
       "evidence": "Section 2.2 states '31% of all repositories.' Validation was performed on 18 SWE-bench repositories (Table 3, Appendix C), with 6-9 out of 18 succeeding depending on configuration.",
       "supported": "moderate"
     },
     {
       "claim": "Performance of some language models on SWE-bench Verified may be inflated due to data contamination.",
       "evidence": "Table 2 shows substantial drops from SWE-bench Verified to SWE-rebench Mar-Apr 2025: DeepSeek-V3-0324 drops from 39.7% to 21.3%, Qwen2.5-72B-Instruct from 11.3% to 9.3%. Both DeepSeek-V3 versions perform similarly on SWE-rebench but diverge on SWE-bench Verified (39.7% vs 35.2%).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4.1 is the only model whose performance noticeably declined between the January and March-April 2025 task subsets.",
       "evidence": "Table 1 shows GPT-4.1 dropping from 31.1% (Jan) to 26.7% (Mar-Apr), while other models generally maintain or improve their resolved rates across the same period.",
       "supported": "moderate"
     },
     {
       "claim": "DeepSeek-V3 models demonstrate the strongest performance among open-source models across both SWE-rebench subsets.",
       "evidence": "Tables 1 and 2 consistently show DeepSeek-V3-0324 and V3-1226 at the top of open-source models: 21.3-21.7% on SWE-rebench Jan and 21.3-21.9% on Mar-Apr, substantially above the next open-source models.",
       "supported": "strong"
     },
     {
       "claim": "Qwen3 models perform similarly with or without think mode enabled.",
       "evidence": "Table 1 shows Qwen3-235B no-think: 15.2%/16.6% vs think: 13.7%/12.2% (Jan/Mar-Apr). Qwen3-32B no-think: 13.2%/13.7% vs think: 11.8%/11.2%. Pass@5 scores are nearly identical. Section 3.3 notes this suggests 'the base model's capabilities are sufficiently strong for deliberate planning to provide no measurable advantage.'",
       "supported": "moderate"
     },
     {
       "claim": "The fine-tuned quality assessment model achieves 81% accuracy on task complexity prediction, improving over the 68% baseline.",
       "evidence": "Section 2.4 reports fine-tuned Qwen-72B achieves 81% accuracy (weighted F1: 0.82) for Task Complexity vs 68% for vanilla Qwen-72B. Table 4 (Appendix F) provides detailed precision/recall breakdowns.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Undisclosed commercial interest",
       "detail": "All authors are from Nebius, which sells AI compute infrastructure (TractoAI). The benchmark promotes the SWE agent training/evaluation ecosystem that drives demand for Nebius infrastructure. This commercial interest is not disclosed as a potential conflict."
     },
     {
       "flag": "Task difficulty confound in contamination claims",
       "detail": "The performance gap between SWE-bench Verified and SWE-rebench (e.g., DeepSeek-V3-0324: 39.7% → 21.3%) is attributed primarily to contamination, but could also be explained by differences in task difficulty, repository diversity, or the automated vs manual curation quality. No formal test controls for these confounds."
     },
     {
       "flag": "Small validation set for pipeline validation",
       "detail": "The automated installation recipe approach was validated on only 18 repositories from SWE-bench (Appendix C), then deployed at scale across 30,000+ repositories. The prompt engineering was also based on this limited validation."
     },
     {
       "flag": "No formal significance testing for model comparisons",
       "detail": "Despite reporting SEM, no formal significance tests are applied to any model comparison claims. With 294 tasks and 5 runs, several reported differences (e.g., Qwen3 think vs no-think) may not be statistically significant."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "The foundational SWE benchmark that SWE-rebench is designed to complement and address contamination issues in."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "Major open-source SWE agent platform evaluated on SWE-bench, directly relevant to agent scaffolding and evaluation practices."
     },
     {
       "title": "Agentless: Demystifying llm-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Agentless approach to SWE tasks that inspired SWE-rebench's installation recipe generation methodology."
     },
     {
       "title": "Training software engineering agents and verifiers with swe-gym",
       "authors": ["Jiayi Pan", "Xingyao Wang", "Graham Neubig"],
       "year": 2024,
       "arxiv_id": "2412.21139",
       "relevance": "Prior dataset for training SWE agents with manually curated interactive tasks, which SWE-rebench aims to surpass in scale."
     },
     {
       "title": "SWE-RL: Advancing llm reasoning via reinforcement learning on open software evolution",
       "authors": ["Yuxiang Wei", "Olivier Duchenne", "Jade Copet"],
       "year": 2025,
       "arxiv_id": "2502.18449",
       "relevance": "Demonstrates RL for training SWE agents, directly relevant to SWE-rebench's stated goal of providing RL training data."
     },
     {
       "title": "SWE-smith: Scaling data for software engineering agents",
       "authors": ["John Yang", "Kilian Leret", "Carlos E. Jimenez"],
       "year": 2025,
       "arxiv_id": "2504.21798",
       "relevance": "Concurrent work on scaling SWE agent training data using artificial bug injection, contrasting with SWE-rebench's real-world task collection."
     },
     {
       "title": "SWE-polybench: A multi-language benchmark for repository level evaluation of coding agents",
       "authors": ["Muhammad Shihab Rashid", "Christian Bock"],
       "year": 2025,
       "arxiv_id": "2504.08703",
       "relevance": "Multi-language SWE benchmark, complementary to SWE-rebench's Python-only scope."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Pioneering continuously-updated decontaminated code benchmark, providing a model for SWE-rebench's temporal decontamination approach."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "The ReAct framework used as the standardized scaffolding for all SWE-rebench benchmark evaluations."
     },
     {
       "title": "Kimi-Dev: Agentless training as skill prior for SWE-agents",
       "authors": ["Zonghan Yang", "Shengjie Wang", "Kelin Fu"],
       "year": 2025,
       "arxiv_id": "2509.23045",
       "relevance": "Work that directly utilizes SWE-rebench dataset for training SWE agents, demonstrating the dataset's practical value."
     },
     {
       "title": "DeepSWE: Training a state-of-the-art coding agent from scratch by scaling RL",
       "authors": ["Michael Luo", "Naman Jain", "Jaskirat Singh"],
       "year": 2025,
       "relevance": "Demonstrates training SWE agents via RL at scale, the primary use case SWE-rebench is designed to support."
     },
     {
       "title": "SWE-bench+: Enhanced coding benchmark for LLMs",
       "authors": ["Reem Aleithan", "Haoran Xue"],
       "year": 2024,
       "arxiv_id": "2410.06992",
       "relevance": "Follow-up work addressing SWE-bench limitations through post-cutoff filtering, contrasting with SWE-rebench's automated approach."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Immediately usable 21K-task dataset on HuggingFace and public leaderboard for practitioners training and evaluating SWE agents."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the validity of SWE-bench Verified scores by showing potential contamination-driven inflation, undermining a widely-used benchmark."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or AI risk concerns raised; brief mention of potential misuse is generic."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Implicitly argues that SWE-bench scores are unreliable due to contamination, a 'benchmarks are inflated' angle that generates discussion."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Dataset on HuggingFace, code on GitHub, and public leaderboard at swe-rebench.com — accessible but not a pip-installable tool."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Nebius is a known AI infrastructure company but not a household name. SWE-bench connection provides some recognition boost."
     }
   }
 }