ai-research-survey

scan-v5.json (19919B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "Defects4C: Benchmarking Large Language Model Repair Capability with C/C++ Bugs",
     "authors": [
       "Jian Wang",
       "Xiaofei Xie",
       "Qiang Hu",
       "Shangqing Liu",
       "Jiongchi Yu",
       "Jiaolong Kong",
       "Yi Li"
     ],
     "year": 2025,
     "venue": "International Conference on Automated Software Engineering",
     "arxiv_id": "2510.11059",
     "doi": "10.1109/ASE63991.2025.00029"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are verified in the paper body: 9M bug-relevant commits, 248 buggy functions, 102 vulnerable functions, and evaluation of 24 LLMs are all confirmed in Tables III–VII.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The fine-tuning causal claim is supported by before/after comparisons using the same models (Table VII), a reasonable design for this type of claim; the paper appropriately hedges that improvements are limited.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Conclusions are consistently bounded to C/C++ single-function bugs from the top 500 GitHub repositories; the paper explicitly states the scope and does not over-generalize to all program repair settings.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The large performance gap between Defects4C and Defects4J is attributed solely to benchmark difficulty, without discussing alternative explanations such as prompt design differences, test harness discrepancies, or different evaluation protocols across the two benchmarks.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses test-passing (pass@k) as the sole measure of 'repair capability' but does not discuss limitations of this proxy, such as overfitting to weak test suites or semantically incorrect but test-passing patches; the distinction between 'plausible' and 'correct' is mentioned only in passing.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII 'Threat to Validity' is a dedicated section covering multiple distinct threats beyond a single sentence.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats include: single-function commit restriction excluding cross-file bugs, top-500 project selection bias, annotation subjectivity with measured Cohen's Kappa values (0.48→0.70→0.88), and training data quality in fine-tuning experiments.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it covers only single-function commits, only the top 500 GitHub C/C++ repositories by stars, and only commits from January 2015 to December 2023; multi-function and cross-file bugs are explicitly excluded.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section IX Acknowledgements discloses funding from NRF Singapore, Cyber Security Agency, CyberSG R&D Programme Office, and Singapore Ministry of Education (RG12/23).",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four author affiliations are listed on the first page: Singapore Management University, Tianjin University, Nanjing University, and Nanyang Technological University.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "All funders are Singapore government agencies (NRF, Cyber Security Agency, MOE) with no commercial interest in which LLMs perform best on C/C++ bug repair.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "The paper contains no competing interests statement or declaration of financial interests such as patents, equity, or consulting relationships.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "APR, single-round repair, conversation-based repair, and line/hunk/function bug granularities are all explicitly defined in Sections I, II, and V; 'plausible' vs 'correct' patches are also distinguished.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper states three explicit contributions in a bulleted list: the Defects4C benchmark dataset, the CLI/API tooling, and the empirical study of 24 LLMs on C/C++ repair.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Table I systematically compares Defects4C against 13 existing C/C++ benchmarks on defect count, project diversity, and source type; Table II empirically motivates the gap via actual LLM performance comparisons.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "Table II demonstrates that contest-style benchmarks yield artificially high LLM scores (GPT-4 at 74.6%) while real-world bugs yield low scores (9.0%), arguing that test-paired real-world bugs better measure actual repair capability.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "Table III categorizes bugs by error type (Signature, Sanitizer, Memory Error, Logic) and counts, but there are no explicit difficulty tiers, no difficulty scores, and no analysis of whether categories differ in expected hardness prior to evaluation.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "GPT-4 achieves only 5/248 repairs (2%) in conversation-based mode, suggesting strong floor effects, but the paper treats this as evidence of benchmark challenge rather than a measurement validity concern; no explicit floor/ceiling analysis is performed.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human expert performance baseline is provided; the paper only reports LLM performance, leaving no reference for whether the benchmark items are solvable at all and at what expected rate.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Pass@k is justified by citing EvalPlus [19] and Chen et al. [18] as established metrics; the unit-test-matching algorithm (Section III-B) that defines which tests count is formally specified with a pass/fail differential criterion.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "For fine-tuning decontamination is performed via UniXcoder cosine similarity filtering, but the evaluation benchmark itself has no contamination-resistance mechanism (no temporal holdouts, canary strings, or dynamic generation) and contamination is only acknowledged as a threat in Section VII.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether the benchmark will remain discriminative as LLMs improve, nor is there a versioning or update plan; temporal coverage of commits (2015–2023) is noted but benchmark longevity is not addressed.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section VII explicitly identifies single-function restriction as a coverage limitation, and RQ3 (Section VI-C, Table VIII) categorizes four failure patterns (long/multi-hunk patches, deletion-centric fixes, missing external context, insufficient test feedback).",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full results for 24 LLMs are reported in Tables IV–VII with exact experimental configurations; the CLI/HTTP API with Docker isolation is publicly released, enabling reproduction of the reported numbers.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Section III provides a comprehensive pipeline description covering raw collection (38M commits), six filtering criteria, the unit-test matching algorithm, and the three-round human annotation protocol with inter-annotator Kappa scores.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "The dataset is described as 'publicly released' with a website link, but no specific license (e.g., MIT, CC-BY, Apache) is stated anywhere in the paper, leaving reuse rights unclear.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly distinguishes Defects4C_bgcommit as suitable for fine-tuning/pretraining (with caveats about false positives) and Defects4C_bug/vul as suitable for rigorous evaluation, with a clear explanation of why each subset serves its intended role.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "State-of-the-art LLMs can fix only 10.88% of general C/C++ bugs and 6.86% of vulnerabilities in Defects4C using conversation-based repair, far below their performance on Java benchmarks.",
       "evidence": "Table IV shows best conversation-based results of 27/248 bugs (10.88%) and 7/102 vulnerabilities (6.86%); Table VI shows Defects4J single-round line repair rates of 71.3% vs single-digit rates on Defects4C.",
       "supported": "strong"
     },
     {
       "claim": "Increasing LLM model size does not consistently improve C/C++ repair performance; CodeLlama-Python improves from 7B to 13B but degrades at 34B.",
       "evidence": "Table V shows CodeLlama-Python pass@100 at T=0.8: 22.5 (7B) → 32.2 (13B) → 29.8 (34B); similar non-monotonic patterns for WizardCoder and CodeLlama-Instruct.",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning LLMs with Defects4C_bgcommit improves C/C++ repair performance by an average of 84.89% relative across 21 of 28 settings.",
       "evidence": "Table VII shows consistent pass@k improvements for fine-tuned CodeLlama-7B-Base (from 0% to 0.41% pass@1 greedy) and CodeLlama-7B-Instruct (2.45%→4.08%); 84.89% average relative improvement is stated in the text.",
       "supported": "moderate"
     },
     {
       "claim": "Contest/interview-style C/C++ benchmarks produce artificially high LLM scores, making them poor proxies for real-world repair capability.",
       "evidence": "Table II shows GPT-3.5 at 94.0% on CodeFlaws and 59.0% on DebugBench (both contest-style) versus 8.5% on Defects4C (real-world).",
       "supported": "strong"
     },
     {
       "claim": "Temperature 0.8 generally produces better single-round repair results than temperature 0.2 across most LLMs.",
       "evidence": "Table V shows consistent higher pass@100 at T=0.8 vs T=0.2 for most models (e.g., GPT-3.5: 38.9 vs 19.5; CodeLlama-Instruct-7B: 45.7 vs 24.9).",
       "supported": "strong"
     },
     {
       "claim": "Long/multi-hunk patches and insufficient test feedback are the dominant failure patterns for LLMs repairing C/C++ vulnerabilities.",
       "evidence": "Table VIII shows 52.0% of failures attributed to long/multi-hunk patches and 9.8% to insufficient test feedback; fine-tuning does not reduce these patterns.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "Defects4C introduces a real-world C/C++ benchmark of 9M bug-related commits, 248 confirmed bugs, and 102 vulnerabilities, revealing that state-of-the-art LLMs fix only 10.88% and 6.86% of these respectively — far below their performance on the Java-based Defects4J benchmark. Larger model size does not consistently improve repair performance, with models producing verbose overgenerated outputs that exceed token limits. Fine-tuning with Defects4C training data yields an average 84.89% relative improvement but still achieves only ~5% pass@1, and fails to address the dominant failure modes of long multi-hunk patches and insufficient test feedback. These findings demonstrate that real-world C/C++ program repair remains a substantially harder and less-solved problem than commonly reported LLM benchmarks suggest.",
   "red_flags": [
     {
       "flag": "No human baseline",
       "detail": "The paper provides no human expert performance data, making it impossible to assess whether benchmark items are achievable in principle or whether floor effects represent a fundamental benchmark problem rather than an LLM limitation."
     },
     {
       "flag": "GPT-4 budget-constrained evaluation",
       "detail": "GPT-4 was limited to only 2 repair attempts (vs 10 for other models) due to cost constraints, making cross-model comparisons in Table IV unfair; the paper acknowledges this but still includes GPT-4 in rankings."
     },
     {
       "flag": "No contamination resistance in benchmark",
       "detail": "The benchmark contains GitHub code from 2015-2023 that may appear in LLM pre-training corpora; the paper dismisses contamination risk by arguing low scores indicate minimal memorization, which is circular reasoning."
     },
     {
       "flag": "No license specified",
       "detail": "The dataset is described as publicly released but no specific license is stated, creating legal ambiguity for downstream research use and reproduction."
     },
     {
       "flag": "Single-function only",
       "detail": "Restricting to single-function commits excludes multi-function and cross-file bugs that may represent a large fraction of real-world defects, limiting ecological validity of the benchmark."
     },
     {
       "flag": "Floor effects not analyzed",
       "detail": "With GPT-4 achieving 2% repair rate on conversation-based mode, the benchmark may be too hard to discriminate among LLMs; this is treated as a feature but could mask measurement noise at the floor."
     }
   ],
   "cited_papers": [
     {
       "title": "Defects4J: A Database of Existing Faults to Enable Controlled Testing Studies for Java Programs",
       "relevance": "Primary comparison benchmark; Defects4C is explicitly designed as the C/C++ analog to Defects4J"
     },
     {
       "title": "Automated Program Repair via Conversation: Fixing 162 out of 337 Bugs for $0.42 Each Using ChatGPT",
       "relevance": "Conversation-based repair method evaluated on Defects4J; Defects4C re-evaluates this approach on C/C++"
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "relevance": "Source of pass@k metric adopted for single-round repair evaluation"
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation (EvalPlus)",
       "relevance": "Evaluation methodology reference for pass@k and greedy decoding protocol"
     },
     {
       "title": "BugsC++: A Highly Usable Real World Defect Benchmark for C/C++",
       "relevance": "Most recent prior work on C/C++ benchmark; Defects4C addresses its limitations (false positives from keyword matching)"
     },
     {
       "title": "The ManyBugs and IntroClass Benchmarks for Automated Repair of C Programs",
       "relevance": "Earlier real-world C benchmark; Defects4C addresses its low usability and outdated C standard support"
     },
     {
       "title": "Magicoder: Source Code is All You Need",
       "relevance": "Decontamination methodology adopted for fine-tuning dataset preparation"
     },
     {
       "title": "Neural Transfer Learning for Repairing Security Vulnerabilities in C Code (VRepair)",
       "relevance": "Keyword-based bug commit filtering methodology adopted from this work"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Security researchers and APR practitioners working on C/C++ can directly use the benchmark and CLI tooling, though the extremely low LLM success rates may limit immediate practical application."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that larger models (34B) perform worse than smaller ones (13B) on C/C++ repair, and that LLMs achieve under 11% even with conversation, challenges the prevailing narrative of LLM coding capability."
     },
     "fear_safety": {
       "score": 2,
       "justification": "C/C++ accounts for over 50% of disclosed open-source vulnerabilities, and the finding that LLMs can fix only 6.86% of vulnerabilities raises concerns about automated security patching claims."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The paper implicitly challenges overly optimistic LLM-for-APR claims but frames this constructively as a benchmark gap rather than as a critique of specific prior work."
     },
     "demo_ability": {
       "score": 2,
       "justification": "The publicly released CLI and HTTP API with Docker-based verification allow researchers to immediately test their own models on the benchmark."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors are from reputable Asian universities (SMU, NTU, Nanjing) but no famous industry lab or flagship model name is behind the work."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "28970112",
         "title": "Stipula: DSL that assists lawyers in programming legal contracts",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=28970112"
       },
       {
         "hn_id": "41866043",
         "title": "Unboxing Virgil ADTs for Fun and Profit",
         "points": 2,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=41866043"
       },
       {
         "hn_id": "37980301",
         "title": "Confidential Consortium Framework: Secure Multiparty Applications",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=37980301"
       }
     ],
     "top_points": 3,
     "total_points": 7,
     "total_comments": 3
   }
 }