ai-research-survey

scan.json (26115B)

---

 {
   "paper": {
     "title": "SWE-MERA: A Dynamic Benchmark for Agentically Evaluating Large Language Models on Software Engineering Tasks",
     "authors": [
       "Pavel Adamenko",
       "Mikhail Ivanov",
       "Aidar Valeev",
       "Rodion Levichev",
       "Pavel Zadorozhny",
       "Ivan Lopatin",
       "Dmitry Babayev",
       "Alena Fenogenova",
       "Valentin Malykh"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2507.11059"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SWE-MERA introduces a dynamic, continuously updated benchmark for evaluating LLMs on software engineering tasks, collecting ~10,000 potential tasks with 300 currently available via a 7-stage automated pipeline from GitHub issues. DeepSeek-R1-0528 achieves the highest pass@6 of 40.2%, with strong discriminative power across model sizes. The paper finds DeepSeek-R1 shows a notable year-over-year performance drop (50% to 40.2% pass@6) compared to other models, suggesting potential contamination effects on older tasks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline released as a Python package on PyPI (https://pypi.org/project/repositorytest) with source code linked. Section 3.2 states 'The entire pipeline is implemented as a Python package and can be executed for any GitHub repository.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset available on HuggingFace (MERA-evaluation/SWE-MERA) as stated in Section 7 submission workflow."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Docker-based execution environment specified with a standardized base image (python:3.11 on Docker Hub). Appendix A shows pip install commands. Conda alternative also mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 provides a submission workflow with steps for dataset acquisition, agent execution, and submission. Appendix A documents the build validation procedure with specific commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows error bars computed using the binomial distribution (5% two-sided quantile), as stated in the figure caption."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims DeepSeek-R1 shows a year-over-year performance difference and ranks models, but no formal significance tests are reported. Confidence intervals are shown visually but no p-values or tests are applied."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results tables report absolute pass rates with baseline context (e.g., DeepSeek-R1 27.8% pass@1, 40.2% pass@6), and year-over-year changes are quantified (e.g., 'decreases from 50% to 40.2%'). This provides enough context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses 300 tasks but no justification is given for why 300 is sufficient for discriminating between models. No power analysis or sample size discussion."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are single-run evaluations per model. While confidence intervals based on binomial distribution are shown, no variance across multiple independent runs is reported. The '3±1 hour' is about execution time, not result variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Ten models evaluated as baselines, including models of varying sizes and types (Table 2), with SWE-bench discussed as a baseline benchmark."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include DeepSeek-R1-0528 (May 2025), Devstral-Small-2505 (May 2025), Qwen3-32B (April 2025) — all contemporary at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study of the pipeline components (e.g., effect of LLM-based filtering, effect of different quality thresholds). The pipeline has 7 stages but their individual contributions are not evaluated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Reports pass@1, pass@6, localize files, generate patch, regression tests, and token limit hit (Table 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of task quality or model outputs. Task quality filtering uses LLM-based evaluation (Qwen3-32B) rather than human judgment."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The dynamic nature ensures tasks are temporally separated from model training data. Tasks are collected monthly from recent GitHub issues (post-September 2024)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by year (2024 vs 2025 tasks in Tables 2 and 4), by model size (Figures 3, 4), and by difficulty decile (Figure 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Discussion section addresses malicious software found in repositories, and the Limitations section discusses failure modes of automated collection (trivial tasks, overly complex tasks, incomplete specifications)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports DeepSeek-R1's unexpected performance degradation on 2025 vs 2024 tasks (Section 5), and discusses cases where automated collection yields unusable tasks (Section 6)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about contamination in SWE-bench (32.67% solution leakage, 31.08% inadequate tests) are attributed to prior work. Claims about 10,000 potential tasks and 300 available are supported by Table 1. 'Strong discriminative power' is supported by the spread of results in Table 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implies DeepSeek-R1's year-over-year performance drop is due to contamination ('performs better on 2024 tasks'), but does not provide causal evidence — the difference could be due to task difficulty differences between years, task distribution shifts, or other confounds."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims evaluation of 'Software Engineering Tasks' but the benchmark is Python-only. The paper acknowledges future expansion to other languages in the conclusion but does not bound current claims to Python."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The DeepSeek-R1 year-over-year performance difference is presented without exploring alternative explanations (task difficulty differences, distribution shifts). The Limitations section discusses general issues but not alternative interpretations of specific results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses pass@1/pass@6 on GitHub issues as a proxy for 'software engineering capability' but does not discuss the gap between resolving curated GitHub issues and actual software engineering ability. The Limitations section acknowledges missing aspects (readability, maintainability, security) but does not frame this as a proxy-outcome distinction."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 lists specific model versions with release dates (e.g., 'DeepSeek-R1-0528', 'Codestral-22B-v0.1', 'Qwen2.5-Coder-32B-Instruct'). Specific git commits for Aider are provided (4f4b10fd, 6e98cd6)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides the full prompt used for LLM-based task evaluation. The Aider framework handles model prompting, and specific Aider versions/commits are cited."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or sampling parameters reported for any of the evaluated models. The paper mentions 6 attempts with 4 reflections per attempt (Aider config) but not model-level generation parameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the Aider framework: 6 independent attempts, up to 4 reflections per attempt for lint/test output, 32k token context limit. Specific Aider commits are referenced."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 documents the full pipeline funnel with counts at each stage (e.g., 255M repos → 300 final tasks), and Section 3.1 describes the criteria at each filtering step."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section with six substantive paragraphs covering task quality, grading challenges, bias, contamination residual risk, infrastructure complexity, and scope limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section raises specific concerns: 'Automatically constructed problems may inadvertently result in unnaturally phrased prompts', 'automated correctness checks may yield false negatives', 'overrepresenting certain programming paradigms.' These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states: 'our current benchmark focuses primarily on programming correctness. Other crucial aspects of software engineering — such as code readability, maintainability, efficiency, security, and teamwork — are not evaluated.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Dataset available on HuggingFace (MERA-evaluation/SWE-MERA), and the pipeline code is released as a Python package, enabling independent verification of the collection process."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the 7-step collection pipeline in detail, including repository selection criteria (10+ stars, 10+ forks, Python, open-source license, recent activity), PR-issue mapping rules, and filtering criteria."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is GitHub repositories with well-documented selection criteria."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 documents each pipeline stage with exact counts (repositories and issues) and time estimates. The funnel from 255M repos to 300 final tasks is fully traced."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section. Authors are from SberAI, ITMO University, and MWS AI, but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: SberAI, ITMO University, MWS AI. These are clearly stated on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed. Authors are affiliated with SberAI (Sber is a major Russian tech company with AI products). The paper does not evaluate Sber models, but the lack of funding disclosure prevents assessment of independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the evaluated models. It relies on temporal separation (tasks from Sept 2024–June 2025) but does not verify this against model training cutoffs."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is motivated by contamination concerns. Section 1 discusses data leakage in SWE-bench. The dynamic collection approach is designed to minimize overlap. The DeepSeek-R1 year-over-year analysis (Section 5) is an indirect test of contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The core contribution is addressing contamination through dynamic updates. Tasks are collected monthly from recent GitHub activity. Section 7 describes the temporal slider for inspecting contamination events. The paper provides its primary raison d'être as contamination mitigation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 reports: '60-140M prompt tokens and 3-20M completion tokens' per model evaluation, '14-20K prompt and 1-4K completion tokens per request', and '3±1 hour' per model evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states '8 NVIDIA H100 80 GB' for most models, '16 GPUs' for DeepSeek-R1, and '4 GPUs' for 7B models. Table 1 provides time estimates for each pipeline stage."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. Each model is evaluated once on the benchmark with no reporting of variance across seeds or runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states Aider gives 6 attempts per task, and reports both pass@1 (first attempt) and pass@6 (any of 6 attempts). However, there is no indication of multiple independent evaluation runs."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates pre-trained models via API/inference without hyperparameter tuning. Aider configuration is fixed."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection — models are evaluated with fixed Aider settings."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple models compared but no correction for multiple comparisons applied. Confidence intervals shown but no formal family-wise error rate control."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the benchmark and evaluate models on it without discussing self-comparison bias or having independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figures 3 and 4 plot pass@1 and pass@6 vs model size, enabling compute/performance comparison. Table 3 lists model sizes alongside results."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether resolving GitHub issues actually measures 'software engineering capability' as claimed. The LLM-based quality filter (Qwen3-32B) is used without validating whether its judgments correlate with human assessments of task quality."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All models evaluated through a single scaffold (Aider). While this controls the scaffold variable across models, the paper does not discuss how Aider-specific design choices might advantage or disadvantage particular models, nor does it evaluate with alternative scaffolds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The core design uses temporal separation — tasks collected from September 2024 to June 2025, after most models' training cutoffs. The year-over-year comparison (Section 5) is an implicit temporal leakage analysis."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 step 3 filters metadata, and SWE-bench+ contamination concerns (solution leakage in issue descriptions) are discussed in Section 2. The LLM-based evaluation step explicitly checks task correctness."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether tasks drawn from the same repositories share structural similarities that could inflate performance. Multiple tasks from the same repo could create non-independence."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The temporal slider in the web interface (Section 7, Figure 2) enables users to 'inspect potential contamination events.' The year-over-year comparison serves as an empirical leakage detection method, revealing DeepSeek-R1's anomalous performance drop."
       }
     }
   },
   "claims": [
     {
       "claim": "SWE-bench has 32.67% of successful patches involving direct solution leakage and 31.08% passing due to inadequate test cases.",
       "evidence": "Abstract, citing SWE-bench+ (Aleithan et al., 2024) findings.",
       "supported": "strong"
     },
     {
       "claim": "The pipeline can collect approximately 10,000 potential tasks, with 300 samples currently available.",
       "evidence": "Table 1 shows the full funnel from 255M repositories to 300 final tasks. The 10,000 estimate is for extending beyond the 6-month window.",
       "supported": "moderate"
     },
     {
       "claim": "DeepSeek-R1-0528 achieves the highest performance with 27.8% pass@1 and 40.2% pass@6 on 2025 tasks.",
       "evidence": "Table 2 results.",
       "supported": "strong"
     },
     {
       "claim": "DeepSeek-R1 shows a larger year-over-year performance decrease (50% to 40.2% pass@6) than other models, suggesting potential contamination.",
       "evidence": "Comparison of Tables 2 and 4, discussed in Section 5 and Appendix C. Figure 5 shows the effect is more subtle on top-decile tasks.",
       "supported": "weak"
     },
     {
       "claim": "SWE-MERA demonstrates strong discriminative power across state-of-the-art models.",
       "evidence": "Table 2 shows a clear spread from 5.5% (Qwen2.5-Coder-7B) to 40.2% (DeepSeek-R1) pass@6.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "LLM-based quality filtering without human validation",
       "detail": "The final quality gate uses Qwen3-32B to score tasks on correctness, test correctness, and completeness. No human validation of these LLM judgments is reported, creating a circular dependency where an LLM evaluates tasks used to evaluate LLMs."
     },
     {
       "flag": "Single scaffold confound",
       "detail": "All models evaluated exclusively through Aider. Since scaffold choice can dramatically affect results (paper's own motivation cites SWE-bench scaffold variance), using only one scaffold limits the generalizability of the model rankings."
     },
     {
       "flag": "Causal contamination claim without adequate evidence",
       "detail": "The paper implies DeepSeek-R1's year-over-year performance drop indicates contamination, but does not control for task difficulty differences between 2024 and 2025 cohorts, distribution shifts, or other confounds."
     },
     {
       "flag": "Python-only benchmark with broad claims",
       "detail": "The benchmark covers only Python repositories but the title and abstract frame it as evaluating 'Software Engineering Tasks' generally."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "The foundational SWE-bench benchmark that SWE-MERA aims to improve upon with dynamic collection."
     },
     {
       "title": "SWE-bench+: Enhanced Coding Benchmark for LLMs",
       "authors": ["Reem Aleithan", "Haoran Xue", "Mohammad Mahdi Mohajer", "Elijah Nnorom", "Gias Uddin", "Song Wang"],
       "year": 2024,
       "arxiv_id": "2410.06992",
       "relevance": "Identified solution leakage and inadequate test cases in SWE-bench, motivating SWE-MERA's design."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Pioneered dynamic, frequently updated evaluation to address contamination, but focused on algorithmic problems rather than repository-level tasks."
     },
     {
       "title": "Multi-SWE-bench: A Multilingual Benchmark for Issue Resolving",
       "authors": ["Daoguang Zan", "Zhirong Huang", "Wei Liu"],
       "year": 2025,
       "arxiv_id": "2504.02605",
       "relevance": "Extended SWE-bench to multiple programming languages, addressing language diversity limitations."
     },
     {
       "title": "Training Software Engineering Agents and Verifiers with SWE-Gym",
       "authors": ["Jiayi Pan", "Xingyao Wang", "Graham Neubig"],
       "year": 2024,
       "arxiv_id": "2412.21139",
       "relevance": "Automatic task generation for training SE agents, complementary approach to SWE-MERA's evaluation focus."
     },
     {
       "title": "SWE-smith: Scaling Data for Software Engineering Agents",
       "authors": ["John Yang", "Kilian Leret", "Carlos E Jimenez"],
       "year": 2025,
       "arxiv_id": "2504.21798",
       "relevance": "Scalable synthetic data creation for SE benchmarks, addressing size and diversity limitations."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Top-performing model on SWE-MERA; its year-over-year performance drop is a key finding."
     },
     {
       "title": "Qwen2.5-Coder Technical Report",
       "authors": ["Binyuan Hui", "Jian Yang", "Zeyu Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Code-specialized LLM family evaluated across multiple sizes, demonstrating scaling behavior on SWE-MERA."
     },
     {
       "title": "Qwen3 Technical Report",
       "authors": ["An Yang", "Anfeng Li", "Baosong Yang"],
       "year": 2025,
       "arxiv_id": "2505.09388",
       "relevance": "Used both as evaluated model and as the LLM quality filter (Qwen3-32B) in the pipeline."
     }
   ]
 }