ai-research-survey

scan.json (26850B)

---

 {
   "paper": {
     "title": "Automated Repair of C Programs Using Large Language Models",
     "authors": ["Mahdi Farzandway", "Fatemeh Ghassemi"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.01947"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 states 'The implementation of our method is accessible on GitHub' and references a GitHub repository, though the specific URL is given only implicitly through the paper's context. The claim is made in the implementation details section."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the Codeflaws dataset (reference [27]), which is a publicly available benchmark. Section 4.4 describes the dataset in detail. Since they used a standard public benchmark without modification, this counts as YES."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using LangChain, LangGraph, GCC with specific flags, and the Suresoft-GLaDOS/SBFL library, but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions. No version numbers are given for any dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the methodology is described at a high level (Section 3), there are no step-by-step reproduction instructions, no README with commands, and no scripts to replicate the main experiments. A researcher would have to reconstruct the pipeline from the prose description."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '44.93% repair accuracy'). No confidence intervals, error bars, or uncertainty measures are provided in Tables 1, 2, or 3."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements over baselines (e.g., '3.61% absolute improvement over GPT-4 with CoT') but provides no statistical significance tests such as p-values, t-tests, or bootstrap tests to support these claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute improvement with baseline context: '44.93% repair accuracy—representing a 3.61% absolute improvement over strong state-of-the-art APR baselines such as GPT-4 with CoT' (41.32%). Table 1 provides baselines and the proposed method's scores, allowing readers to compute relative differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses 3,902 bugs from Codeflaws but there is no justification for why this sample size is adequate, no power analysis, and no discussion of whether the sample is representative of real-world bugs."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. It is unclear whether results represent a single run or are averaged across multiple runs. No seed variation or multiple-run results are mentioned."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares against GPT-4 with CoT (41.32%), SPR (20.06%), CoCoNuT (18.34%), and Angelix (15.14%). Multiple baselines from both classical APR and LLM-based approaches are included."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The classical baselines SPR (2015), Angelix (2016), and CoCoNuT (2020) are quite old. The only contemporary baseline is GPT-4 with CoT. More recent LLM-based APR tools like CigaR (2024), ContrastRepair (2024), and D4C (2024) are discussed in related work but not compared against on the same benchmark. The paper acknowledges not comparing against Agentless and AutoCodeRover due to different task settings."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The five evaluation scenarios (Section 4.2) function as an ablation study: Scenario 1 (no feedback), Scenario 2 (test cases only), Scenario 3 (test cases + SBFL), Scenario 4 (CoT + test cases), Scenario 5 (CoT + SBFL + test cases). Table 1 shows progressive improvement as components are added."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 defines three metrics: Repair Accuracy, Time-to-Repair, and Partial Repair Rate. Results are reported for all three in Tables 1, 2, and 3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated patches is performed. Evaluation is entirely automated (pass/fail on test suites). Human evaluation could have assessed patch quality, readability, or semantic correctness beyond test suite passing."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses the full Codeflaws dataset of 3,902 bugs. There is no mention of train/dev/test splits. Since the LLMs are not fine-tuned on the dataset, a held-out set is less critical, but model selection and hyperparameter tuning (e.g., choosing 4 iterations, Ochiai > 0.5 threshold) are performed on the same dataset used for evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported only as aggregate accuracy across all 3,902 bugs. No breakdown by bug type, defect category, or difficulty level is provided, despite the Codeflaws dataset containing categorization information."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 'Failure Analysis' identifies four categories of failure: non-compilable patches, semantic errors, timeouts/non-terminating executions. This provides substantive analysis of where the approach breaks down."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that approximately 55% of bugs remain unresolved (Section 5.2), and Table 3 shows 48.98% 'No Improvement' cases for the best model. The diminishing returns beyond 4 iterations (Fig 5) is also a negative finding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 44.93% repair accuracy and 3.61% absolute improvement over GPT-4 with CoT. These are supported by Table 1 (Llama 3.1 405b Scenario 5 = 44.93%, GPT-4 CoT = 41.32%, difference = 3.61%)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation-style scenarios (1-5) provide a controlled progression of components. Each scenario adds one element (test feedback, SBFL, CoT), isolating contributions. This controlled single-variable manipulation across scenarios supports the causal claims about which components improve performance."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Automated Repair of C Programs' broadly, but the evaluation is limited to competitive programming bugs from Codeflaws. Section 7 briefly mentions expanding 'beyond C programs from programming competitions to enterprise-level software systems,' but the title and abstract do not bound the claims to competitive programming."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements. For example, the improvement over GPT-4 with CoT could be partly due to the iterative loop (4 iterations vs. single query for GPT-4), different model characteristics, or the specific nature of Codeflaws bugs. Section 7 discusses limitations but not alternative explanations for the results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'Llama 3.1 405b', 'Llama 3.2 90b', and 'Llama 3.1 70b' but does not provide snapshot dates or specific model version identifiers. For the GPT-4 baseline, only 'GPT-4' is mentioned with a reference to the GPT-4 technical report. These are marketing names without version specifics (e.g., no 'meta-llama/Meta-Llama-3.1-405B-Instruct' or similar precise identifier)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figures 4a and 4b show the 'structure' of system and user prompts, but these are schematic diagrams of prompt architecture, not the actual prompt text. The full prompt text used in experiments is not provided in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other LLM API parameters are reported. The paper mentions using OpenRouter platform for API access but does not state any sampling parameters used for the Llama models or the GPT-4 baseline."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 and 4.5 describe the agentic scaffolding in detail: the iterative refinement loop, the memory mechanism for prior attempts, the directed graph-based workflow, the two-tier prompting mechanism, SBFL integration, and the termination conditions. Figure 2 illustrates the workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 describes the Codeflaws dataset structure (faulty implementation, test cases, expected outputs). Section 4.5 describes how code is compiled with GCC flags, how coverage data is collected, and how SBFL scores are computed with the Ochiai formula filtering for scores > 0.5."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations and Future Work' provides a dedicated section discussing limitations including the requirement for executable code, line-level vs. statement-level granularity, dependency on high-quality test cases, and time-related errors."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: 'the current implementation requires executable code to run test cases,' 'our spectral analysis operates at line-level granularity, where statement-level analysis could potentially improve precision,' and 'the evaluation assumes the availability of high-quality test cases, which may not always be available in practice.' These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Section 7 mentions future plans to 'expand beyond C programs from programming competitions to enterprise-level software systems,' the paper does not explicitly state what the results do NOT show. The limitations section focuses on what could be improved, not on what claims cannot be made from the current evidence."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregate results are presented in tables. The per-bug results, individual patch outputs, and detailed repair logs are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 describes the Codeflaws dataset: 3,902 unique code samples from public programming competitions, each consisting of a faulty implementation, test cases with pass/fail results, and expected correct outputs."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The evaluation is entirely computational using the Codeflaws benchmark dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 and 4.5 document the pipeline: code execution with GCC coverage flags, test case execution with 2-minute timeout, coverage data collection via gcov, SBFL computation using Ochiai formula, prompt construction, and iterative refinement. The pipeline from data collection to analysis is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section, no funding disclosure, and no mention of grants or sponsors anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: both authors are from University of Tehran. The paper evaluates open-source Llama models, so there is no direct product-affiliation conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed at all. The absence of a funding disclosure means we cannot assess whether a funder influenced the outcome. The paper uses OpenRouter's free API access for Llama models, which could be considered a form of in-kind support but is not disclosed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the Llama models (3.1 405b, 3.2 90b, 3.1 70b) or GPT-4. This is relevant because Codeflaws bugs come from public programming competitions and could appear in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Codeflaws bugs or their solutions appeared in the training data of Llama or GPT-4. The Codeflaws dataset (2017) predates all models used, making contamination a real concern that is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Codeflaws benchmark was published in 2017, well before the training cutoffs of Llama 3.1/3.2 and GPT-4. Solutions to these competitive programming problems are widely available online. The paper does not acknowledge or address this contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. It is a purely computational benchmark evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports average response times per bug for each model across all scenarios. For example, Llama 3.1 405b takes 80.3 seconds in Scenario 5. While API dollar costs are not reported, the paper notes OpenRouter provided free API access, and time-to-repair is a meaningful practical cost metric."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report total GPU hours, total API calls, total tokens consumed, or total wall-clock time for running all 3,902 bugs across all scenarios and models."
       }
     }
   },
   "claims": [
     {
       "claim": "The approach achieves 44.93% repair accuracy on the Codeflaws benchmark, representing a 3.61% absolute improvement over GPT-4 with CoT.",
       "evidence": "Table 1 shows Llama 3.1 405b Scenario 5 achieves 44.93%, while GPT-4 (CoT) achieves 41.32%. The difference is 3.61 percentage points.",
       "supported": "moderate"
     },
     {
       "claim": "Four iterations represent the optimal configuration for the iterative refinement process.",
       "evidence": "Section 5 and Figure 5 show convergence analysis: 'While we extended our testing to nine iterations, the performance gains beyond the fourth iteration proved negligible.'",
       "supported": "moderate"
     },
     {
       "claim": "Integration of SBFL with CoT reasoning shows synergistic improvement over either component alone.",
       "evidence": "Table 1 shows progressive improvement across scenarios: Scenario 2 (test cases only) < Scenario 3 (+ SBFL) < Scenario 4 (CoT, no SBFL) < Scenario 5 (CoT + SBFL). For Llama 3.1 405b: 36.80% → 41.13% → 42.92% → 44.93%.",
       "supported": "moderate"
     },
     {
       "claim": "Even lower-performing models achieve competitive results with the full methodology.",
       "evidence": "Table 1 shows Llama 3.1 70b in Scenario 5 (38.83%) approaches baseline GPT-4 CoT (41.32%). However, 'competitive' is loosely defined and the gap is still 2.49 percentage points.",
       "supported": "weak"
     },
     {
       "claim": "The system achieves a median repair time of 80 seconds per defect, which is a substantial improvement over existing techniques.",
       "evidence": "Table 2 shows Llama 3.1 405b Scenario 5 averages 80.3 seconds. The paper claims existing techniques require '30 seconds to 4 minutes' but the 80s figure falls within this range, and the high time is partly attributed to API availability issues.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper presents an automated program repair framework that combines spectrum-based fault localization (SBFL) with Chain-of-Thought prompting in an iterative loop for C bug repair. Evaluated on 3,902 Codeflaws bugs, the approach achieves 44.93% repair accuracy using Llama 3.1 405b, a 3.61 percentage point improvement over GPT-4 with CoT (41.32%). The ablation across five scenarios shows each component (test feedback, SBFL, iterative CoT) contributes incrementally to performance, with four iterations found to be optimal before diminishing returns.",
   "red_flags": [
     {
       "flag": "Benchmark contamination risk",
       "detail": "The Codeflaws dataset (2017) is derived from public programming competitions. All models used (Llama 3.1/3.2, GPT-4) were trained well after 2017 and likely saw solutions to these competitive programming problems during training. The paper does not acknowledge or address this contamination risk, which could inflate all reported results including baselines."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "The claimed 3.61% improvement over GPT-4 with CoT is reported without any significance test, confidence interval, or variance estimate. Without knowing the variance across runs, it is impossible to determine whether this difference is meaningful or within random fluctuation."
     },
     {
       "flag": "Unfair baseline comparison",
       "detail": "GPT-4 with CoT is given a single query with test cases and asked to reason and repair, while the proposed method uses 4 iterative queries with SBFL, memory of prior attempts, and runtime feedback. The comparison is between a single-turn approach and a multi-turn agentic approach, making it unclear whether the improvement comes from the methodology or simply from more compute (4x more LLM calls)."
     },
     {
       "flag": "No variance or reproducibility information",
       "detail": "LLM outputs are stochastic, yet the paper reports single-number results with no indication of whether experiments were repeated, what temperature settings were used, or how reproducible the results are."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "No LLM API parameters (temperature, top-p, max tokens) are reported for any model. These significantly affect output and are essential for reproducibility."
     },
     {
       "flag": "Overstated conclusion",
       "detail": "The conclusion states the approach 'achieves a substantial 44.93% absolute accuracy improvement in bug localization and correction for C programs over baseline LLM methods.' This is misleading — the improvement is 3.61 percentage points over GPT-4 with CoT, not 44.93% improvement. The 44.93% is the absolute accuracy, compared to 29.01% for the same model without feedback (Scenario 1)."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational work on LLM code generation capabilities (Codex), directly relevant to evaluating LLM-based programming tools."
     },
     {
       "title": "AutoCodeRover: Autonomous program improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "relevance": "State-of-the-art agentic program repair tool evaluated on SWE-bench, representative of modern AI-based software engineering agents."
     },
     {
       "title": "Keep the conversation going: Fixing 162 out of 337 bugs for $0.42 each using chatgpt",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "Conversational automated program repair using LLMs with cost analysis, directly relevant to LLM-based repair methodology and cost reporting."
     },
     {
       "title": "ContrastRepair: Enhancing conversation-based automated program repair via contrastive test case pairs",
       "authors": ["Jiaolong Kong", "Mingfei Cheng"],
       "year": 2024,
       "arxiv_id": "2403.01971",
       "relevance": "Recent LLM-based program repair approach using contrastive learning with test feedback, closely related methodology."
     },
     {
       "title": "Agentless: Demystifying LLM-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Key work on LLM-based software engineering comparing agentic vs. agentless approaches, directly relevant to survey scope."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E Jimenez", "John Yang"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLM-based software engineering agents on real-world GitHub issues."
     },
     {
       "title": "Automated repair of programs from large language models",
       "authors": ["Zhiyu Fan", "Xiang Gao", "Martin Mirchev", "Abhik Roychoudhury", "Shin Hwei Tan"],
       "year": 2023,
       "relevance": "Empirical study of LLM-generated code repair at ICSE 2023, directly relevant to understanding LLM repair capabilities."
     },
     {
       "title": "CigaR: Cost-efficient program repair with LLMs",
       "authors": ["David Hidvegi", "Khashayar Etemadi", "Sofia Bobadilla", "Martin Monperrus"],
       "year": 2024,
       "arxiv_id": "2402.06598",
       "relevance": "Cost-efficient LLM-based program repair, relevant to evaluating practical cost considerations in AI-based repair tools."
     },
     {
       "title": "Automatic programming: Large language models and beyond",
       "authors": ["Michael R Lyu", "Baishakhi Ray", "Abhik Roychoudhury"],
       "year": 2024,
       "arxiv_id": "2405.02213",
       "relevance": "Survey of LLM capabilities for automatic programming, relevant to understanding the broader landscape of LLM-based code generation."
     },
     {
       "title": "Aligning LLMs for FL-free program repair",
       "authors": ["Junjielong Xu", "Ying Fu", "Shin Hwei Tan", "Pinjia He"],
       "year": 2024,
       "arxiv_id": "2404.08877",
       "relevance": "Alternative approach to LLM-based program repair that aligns model output with repair objectives without fault localization."
     },
     {
       "title": "Copiloting the copilots: Fusing large language models with completion engines for automated program repair",
       "authors": ["Yuxiang Wei", "Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Hybrid approach combining LLMs with completion engines for program repair, relevant to understanding integration of traditional and AI-based repair methods."
     },
     {
       "title": "Fully autonomous programming with large language models",
       "authors": ["Vadim Liventsev", "Anastasiia Grishina"],
       "year": 2023,
       "relevance": "SEIDR system integrating transformers with search-based methods for autonomous programming, relevant to agentic AI for code generation."
     }
   ]
 }