ai-research-survey

scan.json (24831B)

---

 {
   "paper": {
     "title": "BackportBench: A Multilingual Benchmark for Automated Backporting of Patches",
     "authors": [
       "Zhiqing Zhong",
       "Jiaming Huang",
       "Pinjia He"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2512.01396"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/BackportBench/BackportBench. Section 8 (Data Availability) states 'The code and data of BackportBench are available at' this URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (202 task instances with Docker environments) is released at the same GitHub repository. Section 8 confirms code and data availability."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Each task instance is provided as a Docker container with all dependencies installed (Section 3.5: 'all BackportBench task instances are released as Docker containers with all dependencies installed')."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark includes executable Docker environments and test scripts for each task instance. The GitHub repository provides the benchmark suite for reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. All results are point estimates (e.g., '70.8% (143/202)') with no uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., '(M)SWE-agent with Claude Sonnet 4 achieves the highest resolve rate') but provides no statistical significance tests. Comparisons are based solely on raw percentages."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports resolve rates with both percentage and raw counts (e.g., '70.8% (143/202)'), and provides context for comparison such as 'a 15% relative improvement over Mystiqe.' This provides sufficient magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 202 instances but there is no justification for why this size is sufficient for the claims made. Some sub-categories have very small samples (e.g., 3 namespace-only instances, 1 instance per non-file-matched category) with no discussion of statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single runs. No variance, standard deviation, or multiple-run results are reported for any of the evaluated methods."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes multiple baselines: (M)SWE-agent, (M)Agentless, Oracle Retrieval, Mystiqe, and PPatHF. These cover agentic, procedural, oracle, and traditional patch porting methods."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include GPT-5 (2025-09-07), Claude Sonnet 4 (20250514), Qwen3-Coder, SWE-agent, and Agentless, which are all contemporary and competitive systems at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The paper does not test variants of the benchmark construction or the evaluation pipeline. The Oracle Retrieval method provides some insight into the impact of file localization but is not a systematic ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses resolve rate as the primary metric, with breakdowns by FAIL_TO_PASS and PASS_TO_PASS test outcomes, and further categorizes failures into five types (Generation Failed, Only F2P Failed, Only P2P Failed, Both Failed, Timeout)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of system outputs is not applicable here. The benchmark uses execution-based evaluation (test pass/fail), which is more rigorous than human judgment for this task."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark construction paper, not a machine learning training paper. The benchmark itself serves as a test set for evaluating methods."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns are provided: by programming language (Table 3), by content-level category (Table 4), by file-level category (Table 5), and by failure type (Tables 6, 8)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases are analyzed in detail in Tables 6 and 8, with breakdowns by failure type. The paper discusses why PPatHF fails (84.8% generation failures due to misaligned paths) and why Mystiqe fails (74.4% P2P failures due to function-level design)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: all methods perform poorly on logical/structural changes, all methods fail on 'No Overlap' file-level category, Mystiqe's backported patches frequently break existing functionality (74.4% P2P failures), and performance drops sharply for Java and JavaScript compared to Python."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'the agentic method has outperformed traditional patch porting methods, especially on cases that require logical and structural changes' and 'the performance varies across different programming languages.' Both are supported by Tables 3, 4, and 7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes limited causal claims. The suggestion that SWE-agent outperforms due to 'dynamic reflection-and-retry mechanism' (Section 5.1) is an interpretation but is hedged with 'potentially.' Most claims are comparative performance statements supported by controlled experiments on the same benchmark."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its findings to three programming languages (Python, Java, JavaScript) from three ecosystems (PyPI, Maven, npm). Section 6.2 (External Validity) explicitly discusses generalizability limitations, noting that findings depend on the specific packages used."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses multiple alternative explanations: environment mismatch (Section 6.1), data contamination from pre-LLM-cutoff backports (Section 6.1), package-specific maintenance strategies biasing results (Section 6.2), and language-specific LLM performance differences (Section 6.2)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model versions are specified: 'GPT-5-Chat 2025-09-07', 'Claude-Sonnet-4-20250514', and 'Qwen3-Coder-480B-A35B-Instruct' (Section 4.2, Model Selection)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompt adaptations in natural language ('we add the original patches and information from the original codebase... to the prompt') but does not provide the actual prompt text used for any of the methods."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for any of the LLM-based methods. Temperature, top-p, max tokens, and other sampling parameters are not mentioned."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "SWE-agent's scaffolding is described as providing tools to interact with the codebase environment. Agentless's procedural pipeline (hierarchical localization, patch generation, test-based selection) is described. However, details rely on references to the original papers rather than full descriptions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data construction pipeline is documented in detail across Sections 3.1-3.5, including: data source selection (OSV database), deduplication, attribute filtering (3 criteria), manual validation (619 from 1,527 pairs), categorization, and execution-based validation. Specific filtering counts are provided at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Threats to Validity) provides a dedicated discussion with subsections on Internal Validity (6.1) and External Validity (6.2)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: environment mismatch due to loose dependency constraints, data contamination from pre-cutoff backports, package-specific maintenance strategies biasing results, and programming language coverage. These are specific to this study, not generic boilerplate."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 states specific scope boundaries: the benchmark covers only three programming languages, findings depend on the 12 selected packages, and each package follows its own maintenance strategy which could bias results. The paper also notes the benchmark focuses on vulnerability patches specifically."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full benchmark data including task instances, Docker environments, and test cases is available at the GitHub repository. The data collection and curation code is also open-sourced (Section 6.1)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the data collection in detail: OSV database as source, 2024.8.29 data dump, three ecosystems (PyPI, Maven, npm), with specific filtering criteria and counts at each stage (37,284 records to 485 after filtering to 619 confirmed pairs to 202 final instances)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are recruited. The benchmark is constructed from public vulnerability databases and open-source repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Figure 1 and Sections 3.1-3.5: data sources → deduplication/filtering (37,284→485) → commit combination (1,527 pairs) → manual validation (619 confirmed) → repository selection (12 repos, 51.1% coverage) → execution-based validation (202 final instances)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. There is no acknowledgments section mentioning grants or funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are from The Chinese University of Hong Kong, Shenzhen (CUHK-Shenzhen), China. Affiliations are clearly listed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of funding disclosure is a gap."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper acknowledges contamination risk in Section 6.1 ('many of them were conducted predate the LLM's knowledge cutoff date'), the actual training cutoff dates of GPT-5, Claude Sonnet 4, and Qwen3-Coder are not stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 explicitly discusses this: 'due to the scarcity of high-quality backports, many of them were conducted predate the LLM's knowledge cutoff date, which may introduce data contamination into the evaluation results.' They note the open-sourced code facilitates dynamic updates to mitigate this."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 acknowledges contamination risk and provides a mitigation plan: 'we open-source our data collection and curation code to facilitate dynamic updates for BackportBench.' They also plan automated approaches to identify new instances, reducing contamination over time."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation. The manual annotation during benchmark construction is not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs or API expenditures are reported for any of the evaluated methods, despite using multiple commercial LLM APIs (GPT-5, Claude Sonnet 4) across 202 instances with three methods."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper does not report total API spend, GPU hours, or wall-clock time for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "(M)SWE-agent with Claude Sonnet 4 achieves the highest overall resolve rate of 70.8% on BackportBench.",
       "evidence": "Table 3 shows (M)SWE-agent with Claude Sonnet 4 resolving 143/202 instances (70.8%), compared to 69.8% for Oracle Retrieval with GPT-5 and 55.4% for (M)Agentless with GPT-5.",
       "supported": "moderate"
     },
     {
       "claim": "All methods perform significantly worse on Java and JavaScript instances compared to Python instances.",
       "evidence": "Table 3 shows (M)SWE-agent with Claude Sonnet 4 achieving 91.1% on Python vs. 46.3% on Java and 43.5% on JavaScript. This pattern holds across all baseline/model combinations.",
       "supported": "strong"
     },
     {
       "claim": "Agentic methods outperform traditional patch porting methods, especially on cases requiring logical and structural changes.",
       "evidence": "Table 7 shows MSWE-agent with Claude Sonnet 4 resolving 38.5% of logical/structural Java instances vs. Mystiqe at 7.7% and PPatHF at 0.0%. Overall, MSWE-agent achieves 46.3% vs. Mystiqe at 41.8% on Java instances.",
       "supported": "moderate"
     },
     {
       "claim": "Mystiqe's function-level backporting design causes it to break original functionality, as evidenced by 74.4% of failures being P2P test failures.",
       "evidence": "Table 8 shows Mystiqe's failure breakdown: 74.4% (29/39) of failures are 'Only P2P Failed,' meaning backported patches pass vulnerability tests but break existing functionality.",
       "supported": "strong"
     },
     {
       "claim": "Most high-quality backports come from PyPI and Maven ecosystems, with npm having fewer backports despite having more vulnerability records.",
       "evidence": "Section 4.1 reports npm has 30.1% of vulnerability records but only 11.5% of 619 high-quality backports, while PyPI has 27.7% of records but 42.7% of backports.",
       "supported": "strong"
     },
     {
       "claim": "BackportBench is the first comprehensive benchmark suite for patch backporting.",
       "evidence": "Section 2 reviews related work showing existing methods focus on code-hunk or function-level scenarios and use equivalence-based metrics. No prior repository-level, multilingual benchmark with execution-based evaluation is identified.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "BackportBench is a multilingual benchmark of 202 real-world patch backporting problems across Python, Java, and JavaScript with executable Docker environments and test suites. Evaluation of LLM-based methods shows that the agentic approach (SWE-agent) outperforms procedural methods (Agentless) and traditional patch porting methods (Mystiqe, PPatHF), achieving up to 70.8% resolve rate overall but with significant performance variation across programming languages (91.1% Python vs. 43.5-46.3% Java/JavaScript). The benchmark reveals that traditional patch porting methods frequently break existing functionality, a problem hidden by prior equivalence-based evaluation metrics.",
   "red_flags": [
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "All comparative claims are made on raw percentages from single runs without confidence intervals, significance tests, or multi-run variance. Given some sub-categories have very small samples (e.g., 3 namespace-only instances), observed differences may not be statistically meaningful."
     },
     {
       "flag": "Potential selection bias in repository choice",
       "detail": "Only the top 4 repositories per ecosystem by backporting instance count were selected, and those with environment setup failures were replaced. This means the benchmark may over-represent well-maintained packages with good test infrastructure, not representative of the broader ecosystem."
     },
     {
       "flag": "Missing cost reporting",
       "detail": "The paper evaluates 3 methods x 3 commercial LLMs x 202 instances without reporting any API costs or computational budget, making it difficult to assess practical feasibility."
     },
     {
       "flag": "Contamination risk acknowledged but not measured",
       "detail": "The paper acknowledges that many backporting instances predate LLM training cutoffs but does not quantify what fraction of instances may be contaminated or test whether contaminated instances have higher resolve rates."
     }
   ],
   "cited_papers": [
     {
       "title": "Swe-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Foundational benchmark for evaluating LLM agents on repository-level GitHub issue resolution, which BackportBench extends to the patch backporting domain."
     },
     {
       "title": "Swe-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "relevance": "Key agentic baseline evaluated in the paper, representing the state-of-the-art in autonomous LLM-based software engineering agents."
     },
     {
       "title": "Agentless: Demystifying llm-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Key procedural baseline evaluated in the paper, representing the alternative to agentic approaches for automated code modification."
     },
     {
       "title": "Multi-swe-bench: A multilingual benchmark for issue resolving",
       "authors": ["Daoguang Zan", "Zhirong Huang", "Wei Liu"],
       "year": 2025,
       "arxiv_id": "2504.02605",
       "relevance": "Extended SWE-bench to multiple languages; the paper uses MSWE-agent and Magentless from this work for Java and JavaScript evaluation."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational work on LLM code generation evaluation (HumanEval/Codex), establishing benchmarking practices for code-related tasks."
     },
     {
       "title": "Openhands: An open platform for ai software developers as generalist agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "Open platform for AI software engineering agents, relevant to the survey's coverage of agentic AI tools."
     },
     {
       "title": "Autocoderover: Autonomous program improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "relevance": "Autonomous agent for program improvement with AST-based search tools, representing agentic approaches to software engineering."
     },
     {
       "title": "Large language model-based agents for software engineering: A survey",
       "authors": ["Junwei Liu", "Kaixin Wang", "Yixuan Chen"],
       "year": 2024,
       "arxiv_id": "2409.02977",
       "relevance": "Survey of LLM-based agents for software engineering tasks, providing landscape context for this benchmark paper."
     },
     {
       "title": "Mystique: Automated Vulnerability Patch Porting with Semantic and Syntactic-Enhanced LLM",
       "authors": ["Susheng Wu", "Ruisi Wang", "Yiheng Cao", "Bihuan Chen"],
       "year": 2025,
       "relevance": "State-of-the-art patch porting method evaluated as a baseline, using fine-tuned LLMs with semantic/syntactic signatures."
     },
     {
       "title": "Automated patch backporting in Linux (experience paper)",
       "authors": ["Ridwan Shariffdeen", "Xiang Gao", "Gregory J Duck", "Shin Hwei Tan", "Julia Lawall", "Abhik Roychoudhury"],
       "year": 2021,
       "relevance": "Prior work on automated patch backporting using syntactic transformation rules, representing traditional pre-LLM approaches."
     },
     {
       "title": "A Survey on Code Generation with LLM-based Agents",
       "authors": ["Yihong Dong", "Xue Jiang", "Jiaru Qian"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Recent survey on LLM-based code generation agents, relevant to understanding the broader landscape of agentic coding approaches."
     }
   ]
 }