ai-research-survey

scan.json (19832B)

---

 {
   "paper": {
     "title": "A comparative study of large language models with chain-of-thought prompting for automated program repair",
     "authors": ["Eko Darwiyanto", "Rizky Akbar Gusnaen", "Rio Nurtantyana"],
     "year": 2025,
     "venue": "IAES International Journal of Artificial Intelligence (IJ-AI)",
     "doi": "10.11591/ijai.v14.i6.pp4579-4589"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL is provided. The DATA AVAILABILITY section states 'Derived data supporting the findings of this study are available from the corresponding author, [RN], on request,' which does not count as released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The study uses the QuixBugs dataset, which is a publicly available benchmark (reference [13])."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Python and API endpoints but provides no requirements.txt, library versions, or environment specification details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions or README are provided. The workflow is described at a high level (Figures 2 and 3) but without specific commands or scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as averages over 5 runs but no confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CoT prompting 'improves performance' and that DeepSeek-V3 is best, but no statistical significance tests are used. Comparisons are based solely on comparing average numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute improvements with baseline context, e.g., 'GPT-4o records an increase from 31.4 to 35.8 plausible patches' (Section 3.1)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Each model was tested 5 times with no justification for why 5 runs were chosen. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only averages are reported across the 5 runs. No standard deviation, variance, or spread measures are provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Standard (zero-shot) prompting is used as the baseline comparison for CoT prompting across all models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 10 LLM models tested were all released in mid-to-late 2024, which is contemporary for a 2025 paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed on the CoT prompt components (e.g., removing the example section, varying the number of steps). The CoT prompt has multiple components but their individual contributions are not tested."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates both plausible patches (correctness) and token usage cost (efficiency)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. Evaluation is entirely automated via Pytest test cases. Human evaluation could have assessed patch quality beyond test-passing."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the study uses a fixed benchmark (QuixBugs) with no training/tuning phase. The prompts are not tuned on a subset of the data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 (Appendix) provides per-program results for all 40 QuixBugs programs across all 10 models, and Figure 6 breaks down failures by bug type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 identifies failure patterns, discusses the most challenging programs (e.g., lcs_length with 9/10 failures), and analyzes bug types that cause the most failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that o1-mini and o1-preview show decreased performance with CoT prompting, and provides an explanation for why (Section 3.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about DeepSeek-V3 (36.6 patches, $0.006) and GPT-4o (35.8 patches, $0.226) are supported by the results in Section 3. The claim that 'CoT prompting improves performance in most models' is supported, with caveats noted."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CoT prompting 'improves' LLM reasoning ability and 'successfully guides models' (causal language), but the study design is a simple before/after comparison without controlling for confounds like prompt length differences between standard and CoT prompts."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims CoT is 'an effective technique to improve LLM reasoning ability in APR tasks' but the study only tests on the Python portion of QuixBugs (40 small programs). The title says 'automated program repair' generally without bounding to this small benchmark."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the performance improvements, such as the effect of additional prompt length/context in CoT vs standard prompts, or whether the few-shot examples (not CoT steps themselves) drive the improvement."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed by marketing names (GPT-4o, Claude-3.5-Sonnet, etc.) with release dates but without specific API version strings or snapshot dates. 'GPT-4o' released '06 Aug' does not specify which version (e.g., gpt-4o-2024-08-06)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figure 1 shows the prompt structure as a diagram but does not provide the actual prompt text. The exact wording of the CoT steps, examples, and task instructions is not given."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 states 'the temperature is set to 0.0 and top_p to 0.01' using greedy sampling."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system sends single API calls per bug."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4-2.5 describe the APR workflow from reading buggy files to prompt construction to response extraction to validation via Pytest, with flowcharts in Figures 2 and 3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion briefly mentions limitations in one sentence: 'The limitations of this study include the use of a dataset restricted to only the Python version of the QuixBugs dataset, as well as the use of LLM models that are only publicly accessible through APIs.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The brief limitations mention in the conclusion is generic."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the conclusion mentions the dataset is restricted to Python QuixBugs, the paper does not explicitly state what the results do NOT show or what claims cannot be made."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data is 'available from the corresponding author on request,' which does not count as publicly available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is described: QuixBugs benchmark used, each model tested 5 times, responses extracted and validated via Pytest (Sections 2.1, 2.4, 2.5)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is a standard benchmark (QuixBugs)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from reading buggy files to prompt construction to API call to response extraction to Pytest validation is documented with flowcharts (Figures 2 and 3)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "FUNDING INFORMATION section states: 'This research was supported by Directorate of Research and Community Service, Telkom University.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Telkom University and BRIN. No authors are affiliated with the LLM companies being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder (Telkom University) has no financial interest in the outcome of comparing LLM models for APR."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "A CONFLICT OF INTEREST STATEMENT section is present stating: 'Authors state no conflict of interest.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 lists knowledge cutoff dates for some models (e.g., GPT-4o: Oct. 23, Claude-3-5-Sonnet: Apr. 24), though several are marked as unknown ('-')."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "QuixBugs was published in 2017 and is publicly available. All models tested have training data well after 2017, yet no discussion of potential contamination is provided."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "QuixBugs (2017) predates all models' training cutoffs. The benchmark programs are publicly available and likely in training data, but this contamination risk is never discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Token usage costs are reported per model for both prompting methods (Figure 5, Section 3.2), e.g., DeepSeek-V3 at $0.006 and o1-preview at $6.775."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. Individual per-run costs are reported but total API spend for the entire experiment (5 runs x 10 models x 2 methods x 40 programs) is not given."
       }
     }
   },
   "claims": [
     {
       "claim": "CoT prompting improves performance in most LLM models for APR tasks compared to standard prompting.",
       "evidence": "Figure 4 and Section 3.1 show improvements for GPT-4o (31.4 to 35.8), Grok-2 (26.2 to 29.8), DeepSeek-V3, and Llama-3.3-70B, but declines for o1-mini and o1-preview.",
       "supported": "moderate"
     },
     {
       "claim": "DeepSeek-V3 achieves the highest performance with an average of 36.6 plausible patches and the lowest cost of $0.006.",
       "evidence": "Section 3.1 and 3.2 report these numbers directly. Table 2 confirms 37 patches in the best run.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o shows competitive results with an average of 35.8 plausible patches at a cost of $0.226.",
       "evidence": "Reported in Section 3.1 and 3.2.",
       "supported": "moderate"
     },
     {
       "claim": "o1-mini and o1-preview show decreased performance with CoT prompting due to internal CoT mechanisms conflicting with external CoT prompts.",
       "evidence": "Section 3.1 discusses this with reference to OpenAI documentation [28]. The performance decrease is shown in Figure 4, but the causal explanation is speculative.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "This paper compares 10 LLMs on the QuixBugs benchmark using standard vs. chain-of-thought prompting for automated program repair. CoT prompting improved plausible patch counts for most models, with DeepSeek-V3 achieving the best performance (36.6/40 average) at the lowest cost ($0.006). However, models with built-in reasoning (o1-mini, o1-preview) showed decreased performance with external CoT prompting. Failure analysis identified multi-location bugs and missing function calls as the most challenging bug types.",
   "red_flags": [
     {
       "flag": "Benchmark contamination risk",
       "detail": "QuixBugs was published in 2017 and is freely available online. All 10 tested models were trained after 2017, meaning the benchmark programs are very likely in their training data. This is never discussed and could mean the models are 'remembering' fixes rather than reasoning about them."
     },
     {
       "flag": "No statistical rigor",
       "detail": "With only 5 runs and no error bars, confidence intervals, or significance tests, the reported differences between models and prompting methods cannot be distinguished from noise. The difference between 35.8 and 36.6 average patches may not be statistically significant."
     },
     {
       "flag": "Prompts not provided",
       "detail": "The actual prompt text is not given — only a structural diagram (Figure 1). Without the exact prompts, the experiment cannot be reproduced, and the quality of the CoT examples cannot be assessed."
     },
     {
       "flag": "Confound between CoT and few-shot",
       "detail": "The CoT prompt includes two worked examples (few-shot), while the standard prompt is zero-shot. Performance improvements could be due to few-shot examples rather than chain-of-thought reasoning, but this confound is never discussed."
     },
     {
       "flag": "Tiny benchmark",
       "detail": "QuixBugs contains only 40 small single-function programs with simple bugs. Claims about APR effectiveness based on this benchmark have very limited generalizability."
     }
   ],
   "cited_papers": [
     {
       "title": "A survey on automated program repair techniques",
       "authors": ["K. Huang"],
       "year": 2023,
       "arxiv_id": "2303.18184",
       "relevance": "Comprehensive survey of APR techniques including LLM-based approaches."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["J. Wei"],
       "year": 2022,
       "relevance": "Foundational work on CoT prompting used as the basis for this study's approach."
     },
     {
       "title": "Can OpenAI's codex fix bugs?: an evaluation on QuixBugs",
       "authors": ["J. A. Prenner", "H. Babii", "R. Robbes"],
       "year": 2022,
       "doi": "10.1145/3524459.3527351",
       "relevance": "Prior evaluation of LLM (Codex) on QuixBugs benchmark for APR."
     },
     {
       "title": "A survey of learning-based automated program repair",
       "authors": ["Q. Zhang", "C. Fang", "Y. Ma", "W. Sun", "Z. Chen"],
       "year": 2023,
       "doi": "10.1145/3631974",
       "relevance": "Survey of learning-based APR methods published in TOSEM."
     },
     {
       "title": "A comprehensive study of automatic program repair on the QuixBugs benchmark",
       "authors": ["H. Ye", "M. Martinez", "T. Durieux", "M. Monperrus"],
       "year": 2021,
       "doi": "10.1016/j.jss.2020.110825",
       "relevance": "Comprehensive prior study on APR using the same QuixBugs benchmark."
     },
     {
       "title": "Large language models are zero-shot reasoners",
       "authors": ["T. Kojima", "S. S. Gu", "M. Reid", "Y. Matsuo", "Y. Iwasawa"],
       "year": 2023,
       "doi": "10.5555/3600270.3601883",
       "relevance": "Zero-shot CoT prompting technique ('Let's think step by step') used in this study's prompt design."
     },
     {
       "title": "Empirical evaluation of large language models in automated program repair",
       "authors": ["J. Sun", "F. Li", "X. Qi", "H. Zhang", "J. Jiang"],
       "year": 2025,
       "arxiv_id": "2506.13186",
       "relevance": "Recent empirical evaluation of LLMs for APR, directly related to survey scope."
     },
     {
       "title": "DeepSeek-V3 technical report",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2412.19437",
       "relevance": "Technical report for the best-performing model in this study."
     },
     {
       "title": "Towards better chain-of-thought prompting strategies: a survey",
       "authors": ["Z. Yu", "L. He", "Z. Wu", "X. Dai", "J. Chen"],
       "year": 2023,
       "arxiv_id": "2310.04959",
       "relevance": "Survey of CoT prompting strategies referenced for explaining model size effects on CoT performance."
     }
   ]
 }