ai-research-survey

scan.json (31277B)

---

 {
   "paper": {
     "title": "Defects4C: Benchmarking Large Language Model Repair Capability with C/C++ Bugs",
     "authors": [
       "Jian Wang",
       "Xiaofei Xie",
       "Qiang Hu",
       "Shangqing Liu",
       "Jiongchi Yu",
       "Jiaolong Kong",
       "Yi Li"
     ],
     "year": 2025,
     "venue": "International Conference on Automated Software Engineering",
     "arxiv_id": "2510.11059",
     "doi": "10.1109/ASE63991.2025.00029"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLM-based automated program repair techniques fix only 10.88% of general bugs and 6.86% of vulnerabilities in the new Defects4C C/C++ benchmark, dramatically lower than their performance on Java's Defects4J. Fine-tuning on the provided 9M bug-relevant commits improves repair rates by an average of 84.89% relative, but absolute performance remains poor (best pass@1 of 4.92%). Larger models do not consistently outperform smaller variants, and the dominant failure patterns are long/multi-hunk patches (52%) and missing external context (28.4%).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the benchmark is 'publicly released' with a CLI and HTTP interface, accessible at https://sites.google.com/view/defects4c. They describe specific API endpoints (/extract_anchor_patch, /fix_with_patch, /reproduce, /error_dig)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the dataset including 9M bug-relevant commits, 248 buggy functions, and 102 vulnerable functions is publicly released at their project website. They also mention the dataset is 'accessible at the website.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '8X A100-SXM4-80GB GPUs' and Docker containers for patch verification, but does not provide a requirements.txt, Dockerfile for the experiments, or specific library versions. Detailed settings are deferred to the project website."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the paper describes the evaluation workflow, prompt design, and hyperparameters, it does not include step-by-step reproduction instructions for the LLM experiments. Detailed instructions are deferred to the project website."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables V, IV, VI, and VII report only point estimates for pass@k and repair counts. No confidence intervals or error bars are provided on any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims ('LLMs perform better in repairing Defects4C_bug than Defects4C_vul', 'increasing the diversity of model outputs leads to better program repair capability') but no statistical significance tests are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports pass@k scores with sufficient baseline context (e.g., Defects4C_bug 10.88% vs Defects4J 29.8%-71.3%), and relative improvement from fine-tuning ('average relative improvement of 84.89%'). Raw numbers with baselines allow assessment of effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 248 bugs and 102 vulnerabilities. No justification is provided for why these sample sizes are sufficient for the claims being made, nor is any power analysis discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. Pass@k values are reported as single point estimates without any measure of result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 24 LLMs as baselines against each other and compares Defects4C results against Defects4J results from prior work (Table VI). Multiple model families and sizes are compared."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation includes GPT-4, GPT-3.5-Turbo, DeepSeek-Coder, CodeLlama, Gemma, WizardCoder, Magicoder, Mixtral, and Phind-CodeLlama — all contemporary models at the time of the study."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed on the benchmark construction pipeline components (e.g., the effect of keyword filtering, unit test matching algorithm, human annotation). The evaluation examines different strategies (single-round, conversation-based, fine-tuning) but does not ablate their own system."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses pass@1, pass@10, pass@100 for single-round evaluation, and number of successful repairs plus average tries for conversation-based repair."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human annotation is used for dataset construction (validating that commits are genuine bugs), but the evaluation of LLM repair outputs is entirely automated through test case execution. No human evaluation of generated patches is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For fine-tuning experiments (RQ2), the training data (Defects4C_bgcommit) is explicitly decontaminated against the evaluation sets (Defects4C_bug and Defects4C_vul) using UniXcoder embeddings with a 0.95 cosine similarity threshold."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table IV provides per-category breakdowns across Signature, Sanitizer, Memory Error, and Logic Organization categories. Table VI further breaks down by fix granularity (Function, Hunk, Line)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 (Section VI.C) systematically analyzes failure patterns: long/multi-hunk patches, deletion-centric fixes, missing external context, and insufficient test feedback. Table VIII quantifies their distribution."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple negative findings: larger models do not consistently outperform smaller ones (CodeLlama-Python 13B→34B performance drops), fine-tuning improvements are limited (best pass@1 of 4.92%), and open-source models 'perform poorly even in conversation-based repair.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims LLMs have significant limitations on C/C++ repair compared to Java — supported by Tables V, VI. The abstract claims the benchmark enables rigorous evaluation — supported by the construction methodology in Section III. All claims are substantiated in the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Main causal claims include 'fine-tuning benefits repair capability' (supported by controlled before/after comparison in Table VII) and 'increasing diversity of model outputs leads to better repair capability' (supported by temperature comparison). These use controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The title specifies 'C/C++ Bugs' and the paper consistently frames results within the C/C++ domain. Claims about LLM limitations are bounded to 'C/C++ program repair' and compared specifically against Defects4J (Java)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII discusses specific alternative explanations: model contamination could inflate results (but results are low, suggesting minimal effect), annotation subjectivity, training data quality for fine-tuning, and selection bias from focusing on popular projects."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section II explicitly distinguishes between 'plausible' patches (pass all test cases) and 'correct' patches (effectively resolve the underlying bug). The paper reports pass@k and successful repairs, and the measurements match the granularity of claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'GPT-4', 'GPT-35-Turbo', 'CodeLlama-7B', etc. without API snapshot dates or version identifiers. For closed-source models (GPT-4, GPT-3.5-Turbo), no version string (e.g., 'gpt-4-0613') is provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides the complete prompt templates for single function, single hunk, and single line bug types, with concrete examples (parts 4 and 5). The fill values are deterministic from the released dataset (buggy functions and error messages)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V.C reports temperature (0.2, 0.8, 1.0), max token length (2048), conversation parameters (m=10/2, n=3), LoRA rank (8), learning rate (2e-5), batch size (16), epochs (3), and max input length (2048)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The conversation-based repair scaffold is described in Section V.A: iterative invocation, compiler-based verification, error feedback incorporation into prompts, with hyperparameters m (attempts) and n (conversation length). The HTTP verification endpoints are also described in Section IV."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III documents the full pipeline with specific filtering criteria and counts at each stage: 38M commits → 9M (availability validation with 7 criteria) → 76K (single-function filtering) → 3,785 (unit test matching) → 248+102 (human annotation). Each filtering step has explicit criteria."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII 'Threat to Validity' is a dedicated section with five paragraphs discussing specific threats to the study's validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VII discusses study-specific threats: focus on single-function commits excludes multi-function bugs, temporal/contamination bias from popular projects, inter-annotator agreement measured via Cohen's Kappa (0.48→0.70→0.88), and training data quality affecting fine-tuning."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'our focus on single-function commits... excludes multi-function or cross-file defects, such as those involving both a function implementation and its declaration.' They also note plans to 'extend the dataset to include multi-function and cross-file bugs in future releases.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is publicly released at the project website, including bug-relevant commits, buggy functions, and test cases. The CLI provides access to the underlying data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III.A describes collection in detail: BigQuery extraction from 110,441 candidate repos (top 500 by stars, 200+ stars, non-fork, open-source, 2015-2023), keyword-based filtering (fix, solve, repair, bug, etc.), and CVE extraction from CVEProject repository."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Repository selection criteria are explicit (top 500 C/C++ repos by GitHub stars, 200+ stars, open-source with redistributable licenses). For human annotators, qualifications are stated: 'at least 5 years of programming experience and over 3 years in software testing or program analysis.'"
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows the complete pipeline with counts at each stage. Section III details each transformation: 38M→9M (7 filtering criteria), 9M→76K (single-function + test suite), 76K→3,785 (unit test matching), 3,785→248 (three-round human annotation). No unexplained jumps."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section IX (Acknowledgements) discloses funding from the National Research Foundation Singapore, Cyber Security Agency, CyberSG R&D Programme Office, and Singapore Ministry of Education Academic Research Fund Tier 1."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Singapore Management University, Tianjin University, Nanjing University, and Nanyang Technological University. They evaluate third-party LLMs, not their own products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from Singapore government agencies (NRF, CSA, MOE) which are independent research funders with no financial stake in whether LLMs perform well or poorly on C/C++ repair."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 24 LLMs evaluated. This is important since the benchmark commits span 2015-2023 and could overlap with model training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section VII acknowledges 'there is a possibility that similar code patterns may appear in the pre-training corpora of LLMs' but does not perform any analysis (canary strings, membership inference, or temporal splits) to assess actual overlap. They only argue low performance suggests minimal contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark consists of commits from popular GitHub repositories (top 500 by stars, 2015-2023) which are almost certainly in LLM training data. The paper acknowledges this risk in Section VII but does not formally address it beyond noting that low performance suggests contamination effects are 'minimal.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved as study subjects. Human annotators are part of the research team performing dataset construction, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study mines public repositories and evaluates LLMs on code benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects. Annotator qualifications are described but this is for dataset construction, not a human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-example costs are reported. The paper mentions budget constraints limited GPT-4 to 2 repair attempts but does not quantify the actual costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '8X A100-SXM4-80GB GPUs' as hardware but does not report total GPU hours, wall-clock time, or total compute budget for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. Results are presented as single estimates without variation across random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of samples generated per problem for pass@k estimation is not stated. The paper follows EvalPlus methodology but does not specify the exact number of samples n used."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is performed. The paper uses fixed settings (temperatures 0.2, 0.8, 1.0; token limit 2048) adopted from prior work without reporting a search budget or justifying why these specific values were chosen."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports results at multiple temperature settings (0.2, 0.8, greedy) rather than cherry-picking a single best configuration. Results are presented comprehensively across all settings in Table V."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed in the paper, so multiple comparison correction does not apply."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate third-party LLMs on their own benchmark. They do not discuss the potential bias of benchmark constructors choosing bugs that may systematically favor or disfavor certain model capabilities."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4 is limited to 2 repair attempts while other models get 10, creating an unfair comparison. The paper acknowledges 'GPT-4 could achieve higher repair accuracy with more repair attempts' but does not systematically control for compute budget across models."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Table II compares performance on existing C/C++ benchmarks (DebugBench, CodeFlaws) vs Defects4C to demonstrate the difficulty gap between interview/contest-style and real-world bugs. The paper discusses what properties make a benchmark meaningful for APR evaluation."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "For conversation-based repair, the same scaffold (their verification system with identical endpoints) is used across all models. For single-round repair, no scaffolding is involved, so all models are compared directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark commits span 2015-2023. Models trained after 2023 may have seen these commits and their fixes. No temporal analysis is performed to assess this overlap."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation provides error messages from test cases to guide repair. While this is by design, the paper does not discuss whether the amount of information in error messages varies across bugs and could bias results."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "For fine-tuning experiments (RQ2), the paper applies decontamination using UniXcoder embeddings, filtering out training samples with cosine similarity >0.95 to evaluation samples. This explicitly addresses non-independence between train and test data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "For fine-tuning, they use UniXcoder to embed code snippets and filter out samples with cosine similarity >0.95 against evaluation data — a concrete decontamination pipeline. However, no leakage detection is applied for pre-trained model evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM-based APR techniques fix only 10.88% of bugs in Defects4C_bug and 6.86% in Defects4C_vul, significantly lower than Defects4J success rates (29.8%-71.3%).",
       "evidence": "Tables IV, V, and VI show conversation-based repair results: best model repairs 27/248 bugs and 7/102 vulnerabilities. Table VI compares against Defects4J results from Xia et al. showing substantially higher rates on Java.",
       "supported": "strong"
     },
     {
       "claim": "Increasing model size does not necessarily lead to better repair accuracy on C/C++ bugs.",
       "evidence": "Table V shows CodeLlama-Python pass@100 improves from 7B (22.5) to 13B (32.2) but drops at 34B (29.8). Similar trends for WizardCoder-15B/33B and CodeLlama-Instruct sizes.",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuning with Defects4C_bgcommit improves repair performance, with higher pass@k in 21 of 28 cases and an average relative improvement of 84.89%.",
       "evidence": "Table VII compares pre-trained vs fine-tuned results across 4 models. Improvements are seen in most configurations but absolute performance remains low (best pass@1 of 4.92%).",
       "supported": "strong"
     },
     {
       "claim": "Multi-line bugs and bugs requiring external context constitute the largest proportion of LLM failures on Defects4C_vul.",
       "evidence": "Table VIII shows long/multi-hunk patches account for 52.0% and missing external context for 28.4% of failures for CodeLlama-7B-Instruct. Fine-tuning reduces deletion-centric (-2.9%) and external context (-1.9%) failures but not the other types.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs with higher temperature (0.8) usually outperform those with lower temperature (0.2) on the C/C++ APR task.",
       "evidence": "Table V shows systematically higher pass@k values at temperature 0.8 compared to 0.2 across most models (e.g., GPT-3.5 pass@100: 38.9 at T=0.8 vs 19.5 at T=0.2).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Unequal compute budget across models",
       "detail": "GPT-4 is limited to 2 conversation repair attempts due to budget constraints while all other models receive 10. This creates a systematically unfair comparison that the authors acknowledge but do not correct for."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are reported as point estimates without confidence intervals, standard deviations, or error bars. The number of samples used for pass@k estimation is not stated, making it impossible to assess the reliability of reported values."
     },
     {
       "flag": "No training cutoff dates for any model",
       "detail": "None of the 24 evaluated LLMs have their training cutoff dates stated. Since the benchmark commits come from top-500 GitHub repositories (2015-2023), which are almost certainly in LLM training data, this is a significant contamination concern."
     },
     {
       "flag": "Token limit confound",
       "detail": "The 2048-token limit caused 19% of CodeLlama-Instruct-34B outputs to be incomplete. This is an experimental design limitation — the observed 'larger models perform worse' finding may partly be an artifact of output truncation rather than a genuine capability difference."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "The paper makes numerous comparative claims (temperature effects, model size effects, fine-tuning benefits) without any statistical significance testing, despite sufficient data to support such tests."
     }
   ],
   "cited_papers": [
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["C. S. Xia", "Y. Wei", "L. Zhang"],
       "year": 2023,
       "relevance": "Foundational study on LLM-based automated program repair that this work builds upon and extends to C/C++."
     },
     {
       "title": "Automated program repair via conversation: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2024,
       "relevance": "Introduced conversation-based program repair methodology that this paper adapts for C/C++ evaluation."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2023,
       "relevance": "EvalPlus methodology used in this paper for rigorous pass@k evaluation of LLM code generation."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. T. W. Zaremba"],
       "year": 2021,
       "relevance": "Introduced the unbiased pass@k estimation method used throughout this paper's evaluation."
     },
     {
       "title": "Exploring the potential of ChatGPT in automated code refinement: An empirical study",
       "authors": ["Q. Guo", "J. Cao", "X. Xie", "S. Liu"],
       "year": 2024,
       "relevance": "Related empirical study on LLM capabilities for code refinement and repair."
     },
     {
       "title": "Defects4J: A database of existing faults to enable controlled testing studies for Java programs",
       "authors": ["R. Just", "D. Jalali", "M. D. Ernst"],
       "year": 2014,
       "relevance": "The seminal Java defect benchmark that Defects4C is modeled after and compared against."
     },
     {
       "title": "BugsC++: A highly usable real world defect benchmark for C/C++",
       "authors": ["G. An", "M. Kwon", "K. Choi", "J. Yi", "S. Yoo"],
       "year": 2023,
       "relevance": "Most recent prior C/C++ benchmark that Defects4C aims to improve upon."
     },
     {
       "title": "DebugBench: Evaluating debugging capability of large language models",
       "authors": ["R. Tian", "Y. Ye", "Y. Qin"],
       "year": 2024,
       "relevance": "LLM debugging benchmark used as a comparison point showing the gap between contest-style and real-world bugs."
     },
     {
       "title": "Magicoder: Source code is all you need",
       "authors": ["Y. Wei", "Z. Wang", "J. Liu", "Y. Ding", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2312.02120",
       "relevance": "Decontamination methodology adopted for fine-tuning data preparation in this paper."
     },
     {
       "title": "LoRA: Low-rank adaptation of large language models",
       "authors": ["E. J. Hu", "Y. Shen", "P. Wallis"],
       "year": 2021,
       "arxiv_id": "2106.09685",
       "relevance": "Parameter-efficient fine-tuning method used in the paper's RQ2 experiments."
     },
     {
       "title": "ContrastRepair: Enhancing conversation-based automated program repair via contrastive test case pairs",
       "authors": ["J. Kong", "M. Cheng", "X. Xie", "S. Liu"],
       "year": 2024,
       "arxiv_id": "2403.01971",
       "relevance": "Conversation-based APR technique from the same research group evaluating repair with test feedback."
     },
     {
       "title": "UniXcoder: Unified cross-modal pre-training for code representation",
       "authors": ["D. Guo", "S. Lu", "N. Duan"],
       "year": 2022,
       "arxiv_id": "2203.03850",
       "relevance": "Code embedding model used for decontamination in fine-tuning data preparation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The benchmark and CLI tools are directly usable by APR researchers evaluating LLMs on C/C++ code repair."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms the expected difficulty gap between contest-style and real-world bugs; the magnitude of the gap (94% to 8.5% for GPT-3.5) is somewhat surprising."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Highlights that LLMs cannot reliably fix real-world C/C++ vulnerabilities (only 6.86% success), relevant to security but not presenting a novel threat."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Straightforward benchmark paper with no controversy or dramatic claims."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Released benchmark with CLI tools and HTTP API endpoints for automated evaluation, though requires setup."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from Singapore Management University and NTU; evaluates well-known models (GPT-4, CodeLlama) but not from a major AI lab."
     }
   }
 }