ai-research-survey

scan.json (23146B)

---

 {
   "paper": {
     "title": "Automated C/C++ Program Repair for High-Level Synthesis via Large Language Models",
     "authors": [
       "Kangwei Xu",
       "Grace Li Zhang",
       "Xunzhao Yin",
       "Cheng Zhuo",
       "Ulf Schlichtmann",
       "Bing Li"
     ],
     "year": 2024,
     "arxiv_id": "2407.03889",
     "doi": null
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to released code is provided anywhere in the paper. The framework and repair library are described but not made available."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The 24 benchmarks are described as coming from related work [6], [7], [9], [10] and LeetCode, but no dataset download link or archive of the specific benchmark suite is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Intel(R) Xeon(R) Silver 4314 2.40 GHz CPU', Catapult HLS Tool, Synopsys Design Compiler with Nangate 45nm library, and GPT-4 Turbo via OpenAI API, but no software versions, Python environment, or dependency specifications are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts for replicating the experiments are provided. The methodology is described at a conceptual level but not in sufficient detail to reproduce without significant reverse engineering."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as pass rates (percentages) without confidence intervals or error bars. Table II and Figures 6-8 show point estimates only."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the proposed framework 'outperforms the baseline' and achieves 'much higher repair pass rates' but no statistical significance tests are performed to validate these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context: 'an average 23.33% and 13.89% increase in repair pass rate' (Section IV), and specific reductions of '36.57%, 33.03%, and 29.08% reduction in area, power, and minimum clock period' (Section III-C). Per-benchmark pass rates from baseline to proposed are shown in Table II."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Each experiment is repeated n=15 instances, but no justification is given for why 15 was chosen. No power analysis or discussion of whether 15 repetitions is sufficient for reliable pass rate estimation."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only pass rates (m/n with n=15) are reported without standard deviation, confidence intervals, or any spread measure across the 15 runs per benchmark."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table II compares the proposed framework against two baselines: traditional scripts [6][7] and direct application of GPT-4 Turbo for repair."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4 Turbo (2024) is used as the LLM baseline, which was state-of-the-art at submission time. The traditional scripts baseline [6][7] represents the prior non-LLM approach."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 includes an ablation comparing 'GPT-4 Turbo with Scripts' (without RAG) against the full proposed framework (with RAG), demonstrating the RAG contribution. The joint LLM-script mechanism is also evaluated separately in Fig. 7(b)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses compilation pass rate, simulation pass rate, area, power, minimum clock period, and LLM cost as evaluation metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The system produces compilable and simulatable hardware code verified by HLS tools; human evaluation of code quality is not clearly relevant since correctness is verified by C-RTL co-simulation."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning training/testing setup. The benchmarks are real-world C/C++ programs that need repair; there is no train/test split involved."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table II provides per-benchmark results for all 24 applications, organized by 8 error types (T1-T8). Figures 6-8 also show per-benchmark breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper shows that many benchmarks have less than 100% pass rate (e.g., AES at 60%, Edge Detection at 60%) but does not discuss why specific repairs fail, what error patterns are harder, or provide qualitative analysis of failure cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "All experiments show the proposed method improving over baselines. No negative results, failed approaches, or configurations that did not work are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'much higher repair pass rates in 24 real-world applications compared with traditional scripts and direct application of LLMs' and a '23.33% increase in repair pass rate,' both supported by Table II and Section IV results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about RAG improving pass rate (23.33% increase) and joint LLM-script reducing cost (21.56%). The ablation study in Fig. 6 compares GPT-4 Turbo baseline vs. GPT-4 with scripts vs. proposed framework with RAG, providing controlled single-variable comparisons for these components."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Automated C/C++ Program Repair for High-Level Synthesis' broadly, but experiments use only one LLM (GPT-4 Turbo), one HLS tool (Catapult), and 24 benchmarks. No discussion of whether results generalize to other LLMs, HLS tools (Vitis HLS, Intel HLS), or larger programs."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. For example, the improvement could partly be due to the specific correction templates chosen, or the particular benchmarks selected. No threats-to-validity or robustness checks are present."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'GPT-4 Turbo Model was used as the LLM via OpenAI APIs' (Section IV) but does not specify a snapshot date or API version (e.g., 'gpt-4-turbo-2024-04-09'). 'GPT-4 Turbo' is a marketing name without a specific version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides actual prompt text in Figures 10-13 (Appendix II), including system prompts ('You are an expert in HLS repair...'), problem description prompts, and correction template examples. While some content is 'omitted for brevity due to space limitations,' the core prompt structure and actual text are shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, max tokens) are reported. The paper states '5 times to repair' as the iteration limit but no API settings."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail: Section III covers the 5-stage pipeline (preprocessing, RAG-based repair, bit width optimization, equivalence verification, PPA optimization), including the iterative repair loop with compiler feedback, the RAG retrieval mechanism using sentence transformers, and the joint LLM-script interaction."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV states benchmarks come from related work [6][7][9][10], each repeated n=15 instances with 5 LLM queries per instance. The preprocessing stage (Section III-A) describes how programs are first compiled to detect actual errors, then fed to LLM for potential error detection."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations section, threats-to-validity section, or any substantive discussion of limitations in the paper. The conclusion (Section V) contains no caveats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No scope boundaries are stated. The paper does not discuss what settings the results do not apply to, such as other HLS tools, other LLMs, larger or more complex programs, or different error types."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (individual repair attempts, LLM outputs, specific error logs) is available. Only aggregate pass rates are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV describes the experimental setup: 24 benchmarks from related work, GPT-4 Turbo via OpenAI API, Catapult HLS Tool, 15 repetitions per benchmark, 5 LLM queries per repair attempt. Benchmarks are organized by 8 error types."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved. Benchmarks are from prior work and LeetCode, which is a standard data source not involving recruitment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from input C/C++ program through preprocessing, RAG-based repair, bit width optimization, equivalence verification, and PPA optimization is documented in Section III with workflow diagrams (Fig. 2). The evaluation pipeline (compile, simulate, count pass/fail) is described in Section IV."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: TUM, TU Darmstadt, Zhejiang University, and University of Siegen. These are academic institutions, not vendors of the evaluated tools."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The paper uses OpenAI's GPT-4 Turbo and Siemens' Catapult HLS Tool but does not disclose whether any funding relationship exists with these companies."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4 Turbo's training data cutoff is not stated. The benchmarks include LeetCode problems which are publicly available and likely in GPT-4's training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4 Turbo may have seen the benchmark programs or similar HLS repair patterns during training. LeetCode problems ([10]) are widely available online."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmarks include LeetCode problems and examples from prior work published before GPT-4's training cutoff. No contamination analysis is performed despite this clear risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section III-D discusses cost explicitly: GPT-4 Turbo charges '$0.01 per 1K input tokens and $0.03 per 1K output tokens.' Fig. 7(b) shows normalized cost ratios for each error type, and the paper reports a 21.56% average cost reduction with the joint LLM-script approach."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report total API spend, total tokens consumed, or total wall-clock time for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "The proposed LLM-driven framework achieves an average 23.33% increase in repair pass rate compared to the GPT-4 Turbo baseline.",
       "evidence": "Section IV and Fig. 6 compare simulation pass rates across 24 benchmarks. The proposed method with RAG outperforms the direct GPT-4 Turbo baseline by 23.33% on average.",
       "supported": "moderate"
     },
     {
       "claim": "RAG contributes a 13.89% improvement over GPT-4 Turbo with scripts (without RAG).",
       "evidence": "Fig. 6 shows the ablation comparing GPT-4 Turbo with scripts (no RAG) vs. the full proposed framework. The average improvement is 13.89%.",
       "supported": "moderate"
     },
     {
       "claim": "Bit width optimization achieves average reductions of 36.57% in area, 33.03% in power, and 29.08% in minimum clock period.",
       "evidence": "Fig. 7(a) shows normalized performance ratios for 5 benchmarks (ET, GA, BS, CC, AES) after bit width optimization compared to default 32-bit int.",
       "supported": "moderate"
     },
     {
       "claim": "The joint LLM-script repair mechanism reduces LLM usage cost by 21.56% on average.",
       "evidence": "Fig. 7(b) shows normalized cost ratios across 8 error types (T1-T8), comparing LLM-only vs. joint LLM-script repair.",
       "supported": "moderate"
     },
     {
       "claim": "LLM-driven PPA optimization achieves average reductions of 14.02% in area, 11.50% in power, and 17.94% in minimum clock period.",
       "evidence": "Fig. 8 shows normalized performance for 6 benchmarks with performance bottlenecks before and after optimization.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper proposes an LLM-driven framework for automatically repairing regular C/C++ programs to make them compatible with High-Level Synthesis (HLS) tools. Using RAG with a manually-built correction template library, the framework achieves an average 23.33% improvement in repair pass rate over direct GPT-4 Turbo application across 24 real-world benchmarks. An LLM-generated bit width optimization scheme reduces area, power, and clock period by 29-37%, while a joint LLM-script mechanism cuts LLM costs by 21.56%.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "Pass rates are computed from only 15 repetitions per benchmark without confidence intervals, error bars, or standard deviations. With n=15, a single additional success/failure changes the pass rate by 6.67 percentage points, making small differences unreliable."
     },
     {
       "flag": "Potential benchmark contamination",
       "detail": "Several benchmarks come from LeetCode, which is extensively represented in GPT-4's training data. The LLM may have memorized solutions or similar repair patterns, inflating pass rates for both baseline and proposed method. No contamination analysis is performed."
     },
     {
       "flag": "No limitations or threats to validity discussed",
       "detail": "The paper entirely omits any discussion of limitations, threats to validity, or scope boundaries. All results are presented positively with no negative results or failure analysis."
     },
     {
       "flag": "No significance testing for comparative claims",
       "detail": "Claims of superiority ('much higher repair pass rates') are based on comparing raw percentages across 15 trials without any statistical tests. Some reported differences are small enough to be within sampling noise."
     },
     {
       "flag": "Single LLM and single HLS tool",
       "detail": "All experiments use only GPT-4 Turbo and only Catapult HLS, but claims are framed generally ('LLM-driven program repair for HLS'). Generalizability to other LLMs or HLS tools (Vitis HLS, Intel HLS) is unknown."
     }
   ],
   "cited_papers": [
     {
       "title": "Examining Zero-Shot Vulnerability Repair with Large Language Models",
       "authors": ["Hammond Pearce", "Benjamin Tan", "Baleegh Ahmad", "Ramesh Karri", "Brendan Dolan-Gavitt"],
       "year": 2023,
       "relevance": "Evaluates LLMs' ability to automatically repair code vulnerabilities, directly relevant to LLM-based program repair methodology."
     },
     {
       "title": "Repair is nearly generation: multilingual program repair with LLMs",
       "authors": ["Harshit Joshi", "Jose Cambronero", "Sumit Gulwani", "Vu Le", "Ivan Radicek", "Gust Verbruggen"],
       "year": 2023,
       "relevance": "Studies LLM-based multilingual program repair, a core topic in automated program repair with LLMs."
     },
     {
       "title": "ITER: Iterative Neural Repair for Multi-Location Patches",
       "authors": ["He Ye", "Martin Monperrus"],
       "year": 2024,
       "relevance": "Proposes iterative LLM-based repair approach, directly relevant to iterative program repair methodology."
     },
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Comprehensive study of LLM-based automated program repair, core reference for the field."
     },
     {
       "title": "InferFix: End-to-End Program Repair with LLMs",
       "authors": ["Matthew Jin", "Syed Shahriar", "Michele Tufano", "Xin Shi", "Shuai Lu", "Neel Sundaresan", "Alexey Svyatkovskiy"],
       "year": 2023,
       "relevance": "End-to-end LLM-based program repair system, relevant to automated code repair evaluation."
     },
     {
       "title": "AutoChip: Automating HDL Generation Using LLM Feedback",
       "authors": ["Shailja Thakur", "Jason Blocklove", "Hammond Pearce", "Benjamin Tan", "Siddharth Garg", "Ramesh Karri"],
       "year": 2023,
       "arxiv_id": "2311.04887",
       "relevance": "Uses LLM feedback loops for hardware design language generation, relevant to LLM-driven hardware design automation."
     },
     {
       "title": "Chip-Chat: Challenges and Opportunities in Conversational Hardware Design",
       "authors": ["Jason Blocklove", "Siddharth Garg", "Ramesh Karri", "Hammond Pearce"],
       "year": 2023,
       "relevance": "Explores conversational LLM-based hardware design, relevant to LLM applications in hardware/software co-design."
     },
     {
       "title": "GPT4AIGChip: Towards Next-Generation AI Accelerator Design Automation via Large Language Models",
       "authors": ["Yonggan Fu", "Yongan Zhang", "Zhongzhi Yu"],
       "year": 2023,
       "relevance": "Uses GPT-4 for AI chip design automation, relevant to LLM-driven hardware design methodology."
     },
     {
       "title": "Automatically Fixing RTL Syntax Errors with Large Language Model",
       "authors": ["Yun-Da Tsai", "Mingjie Liu", "Haoxing Ren"],
       "year": 2024,
       "relevance": "LLM-based automatic RTL syntax error repair, directly relevant to LLM-driven code repair in hardware design."
     },
     {
       "title": "RTLLM: An Open-Source Benchmark for Design RTL Generation with Large Language Model",
       "authors": ["Yao Lu", "Shang Liu", "Qijun Zhang", "Zhiyao Xie"],
       "year": 2024,
       "relevance": "Benchmark for LLM-based RTL generation, relevant to evaluating LLM capabilities in hardware design."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang", "Ryan Lowe"],
       "year": 2022,
       "relevance": "Foundational RLHF paper for instruction-following LLMs, relevant to understanding the LLM capabilities leveraged in this work."
     }
   ]
 }