ai-research-survey

scan.json (26728B)

---

 {
   "paper": {
     "title": "CigaR: Cost-efficient Program Repair with LLMs",
     "authors": [
       "Dávid Hidvégi",
       "Khashayar Etemadi",
       "Sofia Bobadilla",
       "Martin Monperrus"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2402.06598",
     "doi": "10.48550/arXiv.2402.06598"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/ASSERT-KTH/cigar. Section I states 'We make CIGAR publicly available for future research at https://github.com/ASSERT-KTH/cigar' and Section III-D confirms 'CIGAR and all the data related to our experiments are made publicly available.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks used (Defects4J and HumanEval-Java) are publicly available standard benchmarks. Additionally, Section III-D states 'CIGAR and all the data related to our experiments are made publicly available' including cached prompts, responses, and test results."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions CIGAR is implemented in Python and uses the OpenAI gpt-3.5-turbo-0301 model, but does not provide requirements.txt, Dockerfile, or detailed library version specifications. No environment setup section with dependency versions is present."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include step-by-step reproduction instructions. While the code repository is referenced, the paper itself contains no README-style commands, 'Reproducing Results' section, or specific scripts to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw counts and percentages (e.g., '171/429 (39.8%)') without confidence intervals or error bars. No uncertainty quantification is provided for any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CIGAR 'outperforms' CHATREPAIR and other baselines based solely on comparing raw numbers (e.g., 171 vs 138 correct patches). No statistical significance tests (p-values, bootstrap tests, etc.) are used to validate these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context. For example, '73% cost reduction' (from 467K to 127K tokens per bug), '96% saving' on jointly-fixed bugs (from 608K to 20K), and specific counts for correct patches (171 vs 138). These provide enough context to understand the magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 429 bugs (267 from Defects4J and 162 from HumanEval-Java) without any justification for why this sample size is adequate. No power analysis or discussion of whether 429 bugs is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports single-run results. Due to the temperature=1 setting and stochastic nature of LLM sampling, results could vary across runs, but no variance, standard deviation, or results across multiple seeds are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against three baselines: nl2fix, STEAM, and CHATREPAIR (Table II). CHATREPAIR is reimplemented and compared in detail for both effectiveness (RQ1) and efficiency (RQ2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are all from 2023, which is contemporary for a 2024 paper. CHATREPAIR is described as 'the most advanced APR tool that uses OpenAI's models' based on a literature review (Section IV-C)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "CIGAR has multiple components (initiation prompts, improvement prompts, rebooting, patch multiplication), but no ablation study systematically removes each component to measure its individual contribution. RQ3 studies exploration progress but does not isolate component contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: number of plausible patches, number of correct patches (exact AST match), token cost (total and per-bug average), cost saving percentage, and distinct patch counts."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Correctness is determined entirely by automated AST matching against ground truth. While this is a reasonable automated approach, no human evaluation of patch quality, readability, or semantic correctness beyond test passing is conducted. Given claims about practical usefulness for companies, human evaluation would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Defects4J and HumanEval-Java as evaluation benchmarks. HumanEval-Java is explicitly used as a held-out set to address data leakage concerns: 'this dataset is more recent than the training data used in gpt-3.5-0301, it is not prone to the data leakage problem' (Section VI-A)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table II provides breakdowns by dataset (Defects4J vs HumanEval-Java). Table I shows per-project statistics. Table III breaks down token costs by bug category (fixed by CHATREPAIR only, CIGAR only, both, neither). Table IV provides per-bug analysis for selected bugs."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses unfixed bugs: Table III shows 245 bugs fixed by neither tool with cost analysis. The exploration analysis (RQ3, Figures 5 and 6) discusses cases where long bugs yield few distinct patches during multiplication. Section V-C discusses how bugs with long functions limit patch generation."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows CIGAR improving over baselines. No failed configurations, abandoned approaches, or settings that hurt performance are reported. The paper does not discuss any attempted strategies that did not work."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '73% cost reduction' (supported by Table III: 467K→127K), '96% saving on jointly-fixed bugs' (supported by Table III: 608K→20K), and '171/429 (39.8%)' correct patches (supported by Table II). All numerical claims match the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'CIGAR reduces the token cost by 73%' and 'the rebooting strategy... enables it to efficiently explore different parts of the search space.' These are attributed to CIGAR's design but not rigorously validated with controlled ablations. No component is systematically removed to confirm it causes the observed improvement."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only on Java bugs with a single model (gpt-3.5-turbo-0301) but makes broad claims like 'those original, largely unknown, prompting techniques could be used beyond program repair, in many of the software engineering tasks based on LLMs' (Section V-C). The title says 'Program Repair with LLMs' generically while results are Java-only with one model."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section (Section VI) discusses construct validity (token count as cost metric) and external validity (Java-only), but does not consider alternative explanations for the results. For example, the cost savings could be partly due to CHATREPAIR's inefficient reimplementation rather than CIGAR's design, or the 50-sample batch size being the dominant factor rather than the prompting strategy."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section III-D explicitly states 'CIGAR is implemented in Python and uses OpenAI gpt-3.5-turbo-0301 as the underlying LLM.' This includes the specific version snapshot (0301)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides detailed prompt structures in Figures 2 and 3, showing the initiation prompt and multiplication prompt with their specific sections (system message, one-shot example, buggy code, test failure details, call to action). While these are structural descriptions with example content rather than full verbatim prompt text, the prompts are described at sufficient detail with actual examples shown in the figures."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section III-D reports temperature=1. Section III-B1 states 50 samples per request. Section III-B2 states max_invoke=10 (maximum LLM invocations per round). Section III-C states 5 multiplication invocations. These are the key hyperparameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a detailed workflow diagram (Figure 1) and describes the full pipeline: initiation → partial patch improvement (iterative) → reboot logic → plausible patch multiplication. The feedback mechanism (summarizing previous patches), retry logic (max_invoke=10 invocations), and reboot strategy are all described in Section III."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B and Table I describe how bugs were filtered: only single-function bugs from Defects4J are considered, reducing 596 total bugs to 429. The filtering criterion is explicitly stated. Section III-B1 describes how test results and buggy code are extracted and formatted for prompts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' contains three subsections (Construct Validity, Internal Validity, External Validity) with substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: token count as cost metric may not capture all costs (Section VI-A), data leakage risk for Defects4J with gpt-3.5-0301 (Section VI-A), bug set differences with nl2fix and STEAM (Section VI-B), and Java-only evaluation (Section VI-C). These are particular to CIGAR's experimental setup."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the threats section notes Java-only and Defects4J limitations, the paper does not explicitly state what its results do NOT show. It does not bound its claims to say, e.g., 'our results do not demonstrate cost savings with other LLMs' or 'we do not show our prompting strategies generalize beyond APR.' The conclusion instead suggests broad applicability."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section III-D states 'CIGAR implements a thorough caching system, which stores all the prompts sent to the LLM, all responses received from the LLM as well as the test execution results' and 'CIGAR and all the data related to our experiments are made publicly available.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B describes the bug datasets (Defects4J and HumanEval-Java), how bugs were selected (single-function only), and provides per-project statistics in Table I. Section IV-C describes how baselines were identified (Google Scholar search for GPT-related APR tools)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study uses standard benchmarks (Defects4J and HumanEval-Java), so recruitment methods are not applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: bugs are selected from Defects4J/HumanEval-Java (Table I shows filtering from 596 to 429 single-function bugs), CIGAR runs with cached prompts and responses, and patches are validated via test execution and AST matching. The CHATREPAIR reimplementation process is also described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. There is no mention of grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are identified as being from KTH Royal Institute of Technology, Stockholm, Sweden, with email addresses provided. This is an academic institution without obvious conflicts in evaluating LLM-based APR."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence of the funder cannot be assessed. The absence of funding disclosure means this criterion cannot be satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff for gpt-3.5-turbo-0301. It acknowledges the model 'is likely to have parts of the DEFECTS4J dataset in its training data' (Section VI-A) but does not specify the actual cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-A explicitly discusses data leakage: 'The gpt-3.5-0301 model used by CIGAR is likely to have parts of the DEFECTS4J dataset in its training data.' The paper addresses this by also evaluating on HumanEval-Java, which is 'more recent than the training data used in gpt-3.5-0301.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses contamination risk for Defects4J and mitigates it by using HumanEval-Java as a separate benchmark: 'this dataset is more recent than the training data used in gpt-3.5-0301, it is not prone to the data leakage problem' (Section VI-A). Results on both datasets are compared."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cost reporting is the central contribution of the paper. Table III reports total token costs (204.3M for CHATREPAIR, 54.9M for CIGAR), per-bug averages (467K vs 127K), and per-category breakdowns. Token costs are the primary metric."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While token costs are reported extensively, the total computational budget (wall-clock time, total API spend in dollars, or GPU hours for test execution) is not stated. The paper mentions OpenAI pricing in Section II-B but does not report the actual dollar cost of the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "CIGAR reduces token cost by 73% compared to CHATREPAIR across 429 bugs.",
       "evidence": "Table III shows total token cost of 204.3M for CHATREPAIR vs 54.9M for CIGAR, a reduction of 149.4M/204.3M = 73%. Per-bug average: 467K vs 127K tokens.",
       "supported": "strong"
     },
     {
       "claim": "CIGAR generates correct patches for 171/429 (39.8%) bugs, outperforming all compared LLM-based APR tools.",
       "evidence": "Table II shows CIGAR_R12 fixes 171/429 bugs vs CHATREPAIR's 138/429. However, CHATREPAIR was reimplemented by the authors (Section IV-C), introducing potential implementation bias.",
       "supported": "moderate"
     },
     {
       "claim": "On bugs fixed by both tools, CIGAR saves 96% of token cost (20K vs 608K per bug).",
       "evidence": "Table III 'Both (125)' row shows CHC_AVG=608K, CIC_AVG=20K. The 96% figure is confirmed by the data.",
       "supported": "strong"
     },
     {
       "claim": "CIGAR is the first LLM-based APR tool that focuses on minimizing repair cost.",
       "evidence": "Section VII-A reviews related work and positions CIGAR as 'the first to focus and demonstrate cost minimization without losing effectiveness.' This is a novelty claim about the literature.",
       "supported": "moderate"
     },
     {
       "claim": "The reboot strategy enables CIGAR to efficiently explore different parts of the search space.",
       "evidence": "Figure 5 shows distinct patch counts jumping at reboot boundaries. However, no ablation removes the reboot strategy to measure its isolated contribution vs simply spending more tokens in a single conversation.",
       "supported": "moderate"
     },
     {
       "claim": "46 bugs are uniquely fixed by CIGAR (not fixed by CHATREPAIR).",
       "evidence": "Figure 4 shows the overlap: 125 bugs fixed by both, 46 only by CIGAR, 13 only by CHATREPAIR. These numbers are from the merged dataset.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CigaR is an LLM-based automated program repair tool designed to minimize token cost through three strategies: iterative prompting with patch summarization, search rebooting with high temperature, and plausible patch multiplication. Evaluated on 429 bugs from Defects4J and HumanEval-Java using gpt-3.5-turbo-0301, CigaR reduces token cost by 73% compared to ChatRepair (127K vs 467K tokens per bug on average) while fixing more bugs (171 vs 138 correct patches). On bugs fixed by both tools, CigaR achieves 96% cost savings, spending only 20K tokens per bug versus ChatRepair's 608K.",
   "red_flags": [
     {
       "flag": "Reimplemented baseline",
       "detail": "CHATREPAIR was reimplemented by the authors because the original was not publicly available ('As CHATREPAIR and its patches for DEFECTS4J are not publicly available... we carefully reimplement it'). Reimplementation introduces risk of unintentional handicapping of the baseline, even with private author communication."
     },
     {
       "flag": "No variance across runs",
       "detail": "With temperature=1, LLM outputs are highly stochastic, yet the paper reports single-run results without any variance or confidence measures. The reported numbers could differ substantially across runs."
     },
     {
       "flag": "No ablation study",
       "detail": "CIGAR has three distinct components (improvement prompts, rebooting, patch multiplication) but no ablation study isolates their individual contributions. It is unclear which design choices drive the cost reduction and which are incidental."
     },
     {
       "flag": "No statistical tests for comparisons",
       "detail": "Claims of outperforming baselines rest on comparing raw counts (e.g., 171 vs 138 correct patches) without any statistical significance testing. With stochastic LLM outputs, these differences might not be statistically significant."
     },
     {
       "flag": "Single model evaluation",
       "detail": "All experiments use only gpt-3.5-turbo-0301. The cost minimization strategies are claimed to work for 'any LLM that has an API' but are only tested on one model, limiting generalizability."
     }
   ],
   "cited_papers": [
     {
       "title": "Keep the conversation going: Fixing 162 out of 337 bugs for $0.42 each using chatgpt",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "CHATREPAIR is the primary baseline for both effectiveness and cost comparison in LLM-based program repair."
     },
     {
       "title": "Impact of code language models on automated program repair",
       "authors": ["N. Jiang", "K. Liu", "T. Lutellier", "L. Tan"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00125",
       "relevance": "Large-scale study on using LLMs for APR including HumanEval-Java benchmark used by CigaR for external validity."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Key related work on LLM cost minimization strategies including cascading, prompt adaptation, and model approximation."
     },
     {
       "title": "Cost-effective hyperparameter optimization for large language model generation inference",
       "authors": ["C. Wang", "S. X. Liu", "A. H. Awadallah"],
       "year": 2023,
       "arxiv_id": "2303.04673",
       "relevance": "EcoOptiGen proposes optimizing LLM hyperparameters for cost-effective inference, directly relevant to LLM cost management."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex paper introducing HumanEval benchmark and foundational work on LLM code generation evaluation."
     },
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["C. S. Xia", "Y. Wei", "L. Zhang"],
       "year": 2023,
       "relevance": "Study on few-shot prompting for APR across nine LLMs, foundational work on prompt engineering for program repair."
     },
     {
       "title": "A critical review of large language model on software engineering: An example from chatgpt and automated program repair",
       "authors": ["Q. Zhang", "T. Zhang", "J. Zhai"],
       "year": 2023,
       "arxiv_id": "2310.08879",
       "relevance": "Critical review of LLMs in software engineering with focus on automated program repair prompt engineering."
     },
     {
       "title": "Demystifying gpt self-repair for code generation",
       "authors": ["T. X. Olausson", "J. P. Inala", "C. Wang"],
       "year": 2023,
       "arxiv_id": "2306.09896",
       "relevance": "Introduces pass@t metric for token efficiency in LLM code generation, the cost metric adopted by CigaR."
     },
     {
       "title": "An analysis of the automatic bug fixing performance of chatgpt",
       "authors": ["D. Sobania", "M. Briesch", "C. Hanna", "J. Petke"],
       "year": 2023,
       "arxiv_id": "2301.08653",
       "relevance": "Early analysis of ChatGPT for automated bug fixing, relevant to understanding LLM capabilities in program repair."
     },
     {
       "title": "Automated repair of programs from large language models",
       "authors": ["Z. Fan", "X. Gao", "M. Mirchev", "A. Roychoudhury", "S. H. Tan"],
       "year": 2023,
       "relevance": "Iterative LLM-based APR approach, part of the landscape of LLM repair techniques compared with CigaR."
     },
     {
       "title": "Examining zero-shot vulnerability repair with large language models",
       "authors": ["H. Pearce", "B. Tan", "B. Ahmad", "R. Karri", "B. Dolan-Gavitt"],
       "year": 2023,
       "relevance": "Evaluates LLMs for security vulnerability repair using zero-shot prompts, relevant to LLM code repair capabilities."
     },
     {
       "title": "EcoAssistant: Using LLM assistant more affordably and accurately",
       "authors": ["J. Zhang", "R. Krishna", "A. H. Awadallah", "C. Wang"],
       "year": 2023,
       "arxiv_id": "2310.03046",
       "relevance": "Proposes iterative LLM cost reduction by cascading from cheap to expensive models, related to LLM cost management."
     }
   ]
 }