ai-research-survey

scan-v4.json (20185B)

---

 {
   "scan_version": 4,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "Defects4C: Benchmarking Large Language Model Repair Capability with C/C++ Bugs",
     "authors": [
       "Jian Wang",
       "Xiaofei Xie",
       "Qiang Hu",
       "Shangqing Liu",
       "Jiongchi Yu"
     ],
     "year": 2025,
     "venue": "International Conference on Automated Software Engineering",
     "arxiv_id": "2510.11059",
     "doi": "10.1109/ASE63991.2025.00029"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims LLMs have significant limitations on C/C++ repair compared to Java — supported by Tables V, VI. The abstract claims the benchmark enables rigorous evaluation — supported by the construction methodology in Section III. All claims are substantiated in the results.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Main causal claims include 'fine-tuning benefits repair capability' (supported by controlled before/after comparison in Table VII) and 'increasing diversity of model outputs leads to better repair capability' (supported by temperature comparison). These use controlled single-variable manipulation.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The title specifies 'C/C++ Bugs' and the paper consistently frames results within the C/C++ domain. Claims about LLM limitations are bounded to 'C/C++ program repair' and compared specifically against Defects4J (Java).",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII discusses specific alternative explanations: model contamination could inflate results (but results are low, suggesting minimal effect), annotation subjectivity, training data quality for fine-tuning, and selection bias from focusing on popular projects.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section II explicitly distinguishes between 'plausible' patches (pass all test cases) and 'correct' patches (effectively resolve the underlying bug). The paper reports pass@k and successful repairs, and the measurements match the granularity of claims.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII 'Threat to Validity' is a dedicated section with five paragraphs discussing specific threats to the study's validity.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VII discusses study-specific threats: focus on single-function commits excludes multi-function bugs, temporal/contamination bias from popular projects, inter-annotator agreement measured via Cohen's Kappa (0.48→0.70→0.88), and training data quality affecting fine-tuning.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'our focus on single-function commits... excludes multi-function or cross-file defects, such as those involving both a function implementation and its declaration.' They also note plans to 'extend the dataset to include multi-function and cross-file bugs in future releases.'",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section IX (Acknowledgements) discloses funding from the National Research Foundation Singapore, Cyber Security Agency, CyberSG R&D Programme Office, and Singapore Ministry of Education Academic Research Fund Tier 1.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Singapore Management University, Tianjin University, Nanjing University, and Nanyang Technological University. They evaluate third-party LLMs, not their own products.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from Singapore government agencies (NRF, CSA, MOE) which are independent research funders with no financial stake in whether LLMs perform well or poorly on C/C++ repair.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. Absence of disclosure is not the same as absence of conflict.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined: 'plausible' vs. 'correct' repair (intro), Line/Hunk/Function bug categories (Section IV), single-round vs. conversation-based repair (Section V.A), and pass@k metric (following EvalPlus).",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section I concludes with a bulleted contribution list explicitly claiming: a new executable C/C++ defect benchmark (Defects4C) and a large-scale empirical study of 24 LLMs on C/C++ repair.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section II provides a comparative Table I of 14 prior C/C++ benchmarks and explains specific deficiencies (toy sources, low diversity, poor usability) that Defects4C addresses, situating the contribution directly against existing work.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues construct validity via Table II, showing that interview/contest benchmarks inflate LLM performance (GPT-4: 74.6%) compared to real-world bugs (9.0%), making the case that Defects4C better measures genuine C/C++ repair capability.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "Bug types (Signature, Sanitizer, Memory, Logic) and granularity (Line/Hunk/Function) are catalogued in Table III, but difficulty is not explicitly measured or tiered; the distinction between granularity categories and difficulty levels is not made.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "The benchmark shows severe floor effects (best LLM repairs only 10.88% in conversation mode), but this is framed as 'challenging' rather than explicitly analyzed as a floor effect limiting discriminative power between models.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human performance baseline is reported; the paper only evaluates automated LLM-based approaches, leaving open the question of how this benchmark compares to human-level repair capability.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Pass@k is used following EvalPlus (Liu et al., 2023) and Chen et al. (2021), and repair count is used for conversation-based repair following Xia & Zhang (2024); the metrics are justified by reference to established community practice.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "No structural contamination resistance is built into the evaluation benchmark; the paper acknowledges contamination risk in Section VII but dismisses it post-hoc by noting LLMs underperform, rather than designing temporal splits or canary strings.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions future work extending to multi-function bugs but does not discuss how long Defects4C will remain useful, whether newer LLMs trained on post-2023 data might outperform, or any maintenance plan.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Section VII discusses dataset limitations (single-function scope, false positives), but benchmark failure modes—such as incomplete test suites masking incorrect patches, or gaming the benchmark via test-passing without true repair—are not systematically discussed.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "The paper describes a stateless HTTP and CLI interface with endpoints for patch extraction and isolated Docker-based verification, enabling others to reproduce evaluation numbers; additional details are on the project website.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Section III documents collection sources (GitHub BigQuery, CVEProject), filtering criteria, the unit test matching algorithm, and the three-round human annotation process with inter-annotator agreement metrics; the project website provides additional detail.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "The benchmark is described as 'publicly released' and accessible via a Google Sites URL, but no license for the benchmark itself is specified in the paper; source repo redistribution licenses are only mentioned as a collection criterion.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "Section IV explicitly specifies that Defects4C_bgcommit is for fine-tuning/pretraining while Defects4C_bug and Defects4C_vul are for rigorous evaluation, and the Usage subsection describes the CLI/HTTP interface for large-scale automated evaluation.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "LLM-based APR techniques can only fix 10.88% of bugs in Defects4C_bug and 6.86% in Defects4C_vul in conversation-based repair.",
       "evidence": "Table IV shows best results: 27/248 bugs and 7/102 vulnerabilities repaired by best-performing model (GPT-3.5 Turbo and Phind-CodeLlama respectively).",
       "supported": "strong"
     },
     {
       "claim": "Defects4C is significantly harder than existing C/C++ contest/interview-style benchmarks.",
       "evidence": "Table II shows GPT-3.5 achieves 59% on DebugBench and 94% on CodeFlaws, versus 8.5% on Defects4C under the same settings.",
       "supported": "strong"
     },
     {
       "claim": "Larger model size does not consistently improve APR performance on Defects4C.",
       "evidence": "Table V shows CodeLlama-Python pass@100 improves from 22.5 (7B) to 32.2 (13B) then drops to 29.8 (34B); similar patterns in WizardCoder and CodeLlama-Instruct.",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning with Defects4C_bgcommit improves repair performance by an average relative improvement of 84.89%.",
       "evidence": "Table VII shows improvements in 21/28 cases; CodeLlama-7B-Instruct improves from 2.45 to 4.08 pass@1 under greedy decoding after fine-tuning.",
       "supported": "moderate"
     },
     {
       "claim": "Long/multi-hunk patches and missing external context are the dominant failure patterns for LLMs on Defects4C_vul.",
       "evidence": "Table VIII shows 52% of failures are long/multi-hunk patches and 28.4% are missing external context for CodeLlama-7B-Instruct.",
       "supported": "moderate"
     },
     {
       "claim": "C/C++ program repair remains significantly more challenging for LLMs than Java repair (Defects4J).",
       "evidence": "Table VI shows Defects4J repair rates of 29.8–71.3% vs. Defects4C rates of 0–13.6% for the same models and repair strategies.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "benchmark-creation"
   ],
   "key_findings": "Defects4C fills a gap in C/C++ program repair research by providing 350 human-verified bugs and vulnerabilities from 41 real-world projects, paired with executable test cases and a usable CLI/API. State-of-the-art LLMs repair only 10.88% of general bugs and 6.86% of vulnerabilities in conversation-based mode, far below their performance on Java (Defects4J) and existing contest-style C/C++ benchmarks. Larger models do not consistently outperform smaller ones, with verbosity and token-limit issues degrading larger model output. Fine-tuning on the 9M-commit training corpus yields modest average improvement (~85% relative) but absolute pass@1 rates remain below 5%, indicating that C/C++-specific advances are needed.",
   "red_flags": [
     {
       "flag": "No human baseline",
       "detail": "The benchmark evaluates only automated LLM approaches; no human performance baseline is reported, making it impossible to interpret whether the benchmark's difficulty is appropriate or whether it is unsolvable even for humans."
     },
     {
       "flag": "Floor effects not addressed",
       "detail": "Most models repair fewer than 5% of bugs at pass@1, yet ceiling/floor effects are not analyzed. At these rates the benchmark has limited power to discriminate between models."
     },
     {
       "flag": "GPT-4 budget artificially constrained",
       "detail": "GPT-4 is limited to 2 repair attempts in conversation-based evaluation due to cost, while other models get 10 attempts. This makes the cross-model comparison in Table IV misleading for GPT-4."
     },
     {
       "flag": "Contamination dismissed without rigorous analysis",
       "detail": "Contamination risk is acknowledged in Section VII but dismissed by ex-post reasoning ('LLMs underperform, so contamination must be minimal') rather than through designed temporal holdouts or canary strings."
     },
     {
       "flag": "Benchmark license unspecified",
       "detail": "The paper provides no license for the benchmark itself; reuse terms are unclear despite the benchmark being described as publicly available."
     },
     {
       "flag": "Test-passing conflated with correctness",
       "detail": "Pass@k on the provided test suite is used as a correctness proxy throughout, but incomplete test suites could pass incorrect patches; the plausible-vs-correct distinction raised in the intro is not revisited in results."
     }
   ],
   "cited_papers": [
     {
       "title": "Defects4J: A Database of Existing Faults to Enable Controlled Testing Studies for Java Programs",
       "relevance": "The primary inspiration and comparison benchmark for Defects4C; the Java APR benchmark that this work is explicitly designed to replicate and extend for C/C++."
     },
     {
       "title": "Automated Program Repair via Conversation: Fixing 162 out of 337 Bugs for $0.42 Each Using ChatGPT",
       "relevance": "Provides the conversation-based APR methodology and Defects4J baseline numbers used directly for comparison in Table VI."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation (EvalPlus)",
       "relevance": "Source of the pass@k evaluation metric and evaluation methodology used throughout the empirical study."
     },
     {
       "title": "BugsC++: A Highly Usable Real World Defect Benchmark for C/C++",
       "relevance": "The most recent prior C++ benchmark; directly compared and critiqued for including non-genuine bugs in commit messages."
     },
     {
       "title": "The ManyBugs and IntroClass Benchmarks for Automated Repair of C Programs",
       "relevance": "Key prior C APR benchmark compared against Defects4C; critiqued for low usability and limited diversity."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "relevance": "Introduces pass@k metric and establishes baseline methodology for LLM code evaluation used in this paper."
     },
     {
       "title": "Magicoder: Source Code Is All You Need",
       "relevance": "Provides the decontamination methodology using UniXcoder similarity filtering applied to the fine-tuning dataset."
     },
     {
       "title": "UniXcoder: Unified Cross-Modal Pre-Training for Code Representation",
       "relevance": "The model used for similarity-based decontamination of the fine-tuning split from the evaluation set."
     },
     {
       "title": "Neural Transfer Learning for Repairing Security Vulnerabilities in C Code (VRepair)",
       "relevance": "Provides the keyword-based commit filtering heuristic adapted for collecting bug-related commits from GitHub."
     },
     {
       "title": "DBGBench: Where Is the Bug and How Is It Fixed? An Experiment with Practitioners",
       "relevance": "Existing real-world C bug benchmark compared against Defects4C; critiqued for covering only 2 projects."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The benchmark and CLI tools are directly usable by APR researchers evaluating LLMs on C/C++ code repair."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms the expected difficulty gap between contest-style and real-world bugs; the magnitude of the gap (94% to 8.5% for GPT-3.5) is somewhat surprising."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Highlights that LLMs cannot reliably fix real-world C/C++ vulnerabilities (only 6.86% success), relevant to security but not presenting a novel threat."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Straightforward benchmark paper with no controversy or dramatic claims."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Released benchmark with CLI tools and HTTP API endpoints for automated evaluation, though requires setup."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from Singapore Management University and NTU; evaluates well-known models (GPT-4, CodeLlama) but not from a major AI lab."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "28970112",
         "title": "Stipula: DSL that assists lawyers in programming legal contracts",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=28970112"
       },
       {
         "hn_id": "41866043",
         "title": "Unboxing Virgil ADTs for Fun and Profit",
         "points": 2,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=41866043"
       },
       {
         "hn_id": "37980301",
         "title": "Confidential Consortium Framework: Secure Multiparty Applications",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=37980301"
       }
     ],
     "top_points": 3,
     "total_points": 7,
     "total_comments": 3
   }
 }