ai-research-survey

scan.json (31720B)

---

 {
   "paper": {
     "title": "Exploring Code Language Models for Automated HLS-based Hardware Generation: Benchmark, Infrastructure and Analysis",
     "authors": [
       "Jiahao Gai",
       "Hao (Mark) Chen",
       "Zhican Wang",
       "Hongyu Zhou",
       "Wanru Zhao",
       "Nicholas Lane",
       "Hongxiang Fan"
     ],
     "year": 2025,
     "venue": "ASP-DAC'25 (Asia and South Pacific Design Automation Conference)",
     "arxiv_id": "2502.13921",
     "doi": "10.1145/3658617.3697616"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL is provided for the framework, fine-tuned models, or evaluation infrastructure. The paper references open-source tools used (axolotl, Code-Llama) but does not release its own code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The collected dataset of 42,000+ HLS programs is not released. The paper describes collecting from HLSyn and ML4Accel repositories but does not provide a download link for the curated and filtered dataset or the test set with unit tests."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 lists hardware (4x NVIDIA L20 GPUs, 80 vCPU Intel Xeon Platinum 8457C, 100GB RAM) and some training settings (8-bit loading, sequence length 4096), but no requirements.txt, Dockerfile, or complete library version list is provided. Software versions are incomplete (Vivado 2020.1 is mentioned but Python version, PyTorch version, axolotl version, etc. are not)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the framework at a high level but does not include commands, scripts, or a README for replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as single point estimates (e.g., 88.44% syntax, 53.20% functionality in Figure 6). No confidence intervals or error bars appear in any figure or table."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements from finetuning, CoT, and feedback loops based solely on comparing raw percentages. No statistical significance tests (t-tests, bootstrap, etc.) are applied to any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with both baseline and improved values, providing context for the magnitude of effects. For example, Section 5.2: syntax from 54.85% to 88.44%, functionality from 0% to 53.20%. Section 5.3: syntax from 88.44% to 94.33%, functionality from 53.20% to 61.45%."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test set derives from a 4:1 split of only 52 base designs (Section 3.2), yielding approximately 10 test designs. No justification is given for why this sample size is sufficient. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported for any experiment. All results appear to be from single runs without repeated trials."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The non-finetuned Code-Llama-7B serves as a baseline (Section 5.2, Figure 6a). Ablations of CoT and feedback loops provide additional internal baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "All comparisons are against ablations of the authors' own system. No external baselines are included — no comparison with VeriGen, VerilogEval methods, RTLFixer, LLM-VeriPPA, or other contemporary LLM-for-hardware approaches discussed in Section 2.2."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Sections 5.2–5.4 systematically ablate three components: supervised fine-tuning (Section 5.2), chain-of-thought prompting (Section 5.3), and feedback loops with varying iterations (Section 5.4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used throughout: pass@3 syntax correctness and pass@3 functionality correctness. Table 1 also reports latency and resource usage (LUTs, registers, DSP48s, BRAMs) for synthesized designs."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code quality. All evaluation is automated through GCC syntax checking and unit test comparison."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 states the 52 designs are 'split into training and testing sets at a 4:1 ratio.' Results are reported on this test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 (Section 5.6) breaks down performance by complexity level (easy, medium, difficult). Table 3 (Section 5.7) compares MachineGen vs HumanRefine. Section 3.2 categorizes designs into five types."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses performance trends (e.g., decline with complexity, HumanRefine gap) but does not show specific examples of failed generations, common error types, or qualitative analysis of what goes wrong."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports the dramatic HumanRefine performance drop (syntax 47.29%, functionality 21.36% vs MachineGen's 93.83%, 62.24%). Section 5.4 reports diminishing returns for the second feedback loop iteration. Section 5.8 acknowledges benchmark diversity limitations."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'Comprehensive experiments demonstrate the effectiveness of our methods' and the results in Sections 5.2–5.4 do show improvements from finetuning, CoT, and feedback loops. The abstract is relatively hedged, noting limitations and future work."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims (e.g., 'finetuning dramatically increases syntax correctness,' 'incorporating CoT leads to a noticeable improvement') are backed by controlled ablation studies that vary one component at a time (Sections 5.2–5.4). The ablation design is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Automated HLS-based Hardware Generation' broadly, but experiments use only one model (Code-Llama-7B), one fine-tuning method (QLoRA), and 52 base designs from two repositories. Section 5.8 acknowledges 'the diversity of hardware designs in the benchmark is limited' but the title and abstract do not bound claims to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.7 offers one interpretation for the MachineGen/HumanRefine gap (training data bias, complexity of human prompts) but does not consider alternatives. No discussion of confounds such as whether improvements from CoT are due to longer input sequences, or whether the non-finetuned model's 0% functionality could reflect prompt format mismatch rather than lack of HLS knowledge."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@3 for syntax correctness and functionality correctness, and claims are framed in terms of these specific metrics. No broader claims about 'hardware design quality' beyond what is measured, though Section 5.8 notes hardware performance feedback is left for future work."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'Code-Llama-7B' (Section 5.1) and 'ChatGPT (version 3.5 and 4)' (Section 3.3) but provides no specific checkpoint, snapshot date, or API version for any model. Code-Llama-7B has multiple releases, and 'ChatGPT 3.5 and 4' are marketing names without version granularity."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The default instruction prompt is stated as 'Generate HLS code with the following instructions:' (Section 3.1). Figure 5 provides the full CoT prompt text ('Let's think step by step. First, Consider the characteristics of FPGA...'). These are the actual prompts used, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 reports some settings: 8-bit loading, sequence length 4096, warmup steps 100, gradient accumulation 4, micro-batch size 4, inference batch size 2. However, critical hyperparameters are missing: learning rate, number of epochs, temperature and sampling settings for generation (top-p, top-k)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The iterative framework is described in detail: Figure 4 provides a full overview, Section 4.1 describes the two-stage pipeline, Section 4.2 details the CoT mechanism, and Section 4.3 describes the two-step feedback loop (syntax check → feedback → functionality check → feedback) with iteration control."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.2 states designs were collected from HLSyn and ML4Accel, then 'We filter out the HLS programs that are invalid, resulting in a collection of over 42,000 HLS programs.' The filtering criteria for 'invalid' programs are not specified, and the count before filtering is not stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.8 'Thoughts, Insights, and Limitations' contains a dedicated subsection (item 3) discussing limitations including unavailability of advanced reasoning models, limited benchmark diversity, and missing test-time scaling approaches."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.8 item 3 discusses specific threats: DeepSeek-R1 was not available for evaluation, test-time scaling could improve results, and 'the diversity of hardware designs in the benchmark is limited, which may impact the generalizability of our findings.' These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it focuses on C-based HLS (footnote Section 2), acknowledges the benchmark's limited diversity may impact generalizability (Section 5.8), and identifies hardware performance feedback as outside current scope. The abstract notes limitations due to 'the timeframe of this research.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the collected HLS dataset, the generated design descriptions, the fine-tuned model weights, nor the test outputs are made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the data sources (HLSyn, ML4Accel open-source repositories), the five design categories (matrix operations, scientific simulations, statistical computations, iterative methods, other kernels), the filtering of invalid programs, and the 4:1 train/test split."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from identified open-source repositories (HLSyn, ML4Accel)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline goes from 52 base designs → different pragma combinations → filtering invalid programs → 42,000+ programs → 4:1 split. However, the number of programs before filtering is not stated, the criteria for 'invalid' are unspecified, and the exact test set size is not given."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information appears anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Imperial College London, University of Cambridge, Shanghai Jiao Tong University, University of Sydney. All are academic institutions with no apparent product conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making it impossible to assess funder independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper fine-tunes and evaluates Code-Llama-7B on an HLS benchmark but never states the model's pre-training data cutoff date. This is relevant because the HLS designs come from public GitHub repositories that may be in the pre-training corpus."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the HLS designs from HLSyn and ML4Accel (both on GitHub) could have appeared in Code-Llama's pre-training data. The train/test split is described but potential overlap with pre-training data is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark designs are sourced from public GitHub repositories (HLSyn, ML4Accel) that existed before Code-Llama's training. Code-Llama was trained on GitHub data, creating a clear contamination risk that is never discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of LLM-generated HLS code."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of LLM-generated HLS code."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of LLM-generated HLS code."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of LLM-generated HLS code."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of LLM-generated HLS code."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of LLM-generated HLS code."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of LLM-generated HLS code."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 9 reports inference time averaged over 120 data points under different conditions (w/o feedback, syntax loop, functionality loop, each with and without CoT). Time cost ranges from 5–11 seconds per data point."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 describes the hardware setup (4x NVIDIA L20 GPUs, 80 vCPU, 100GB RAM) but does not state total GPU hours for fine-tuning, total training time, or overall computational budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from a single training run and single evaluation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses pass@3 (3 generated samples per problem) but does not state how many independent runs of the entire experiment were conducted. It is unclear whether results are from a single run or averaged across multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 lists fixed hyperparameter values but does not describe any hyperparameter search process, number of configurations tried, or how the reported values were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports results for specific configurations without explaining how they were selected. No validation-based selection process is described."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so the question of correcting for multiple comparisons is structurally inapplicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All comparisons are between the authors' own system variants. No external baselines are included and no discussion of author-evaluation bias is present."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Figure 9 shows time cost for different configurations, but performance is not plotted as a function of compute budget. The time and accuracy results are presented separately, without matched-compute analysis."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether pass@3 syntax correctness and unit-test-based functionality adequately measure the ability to generate useful hardware designs. No discussion of construct validity or comparison with alternative evaluation approaches."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The framework (scaffold) IS the thing being evaluated and ablated. The paper tests one model (Code-Llama-7B) within its own scaffold, varying scaffold components as independent variables. No cross-model-cross-scaffold comparison is made."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The HLS designs sourced from HLSyn and ML4Accel were publicly available on GitHub before Code-Llama's pre-training. No discussion of whether the model could have memorized these designs during pre-training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides any information leakage. The MachineGen prompts are generated by ChatGPT from the reference designs — the degree of information transferred from reference to prompt is not analyzed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Training and test sets are derived from the same 52 base designs with different pragma combinations (PIPELINE, PARALLEL, TILE). This creates high structural similarity between train and test examples. This non-independence is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, n-gram analysis, or decontamination pipeline is mentioned."
       }
     }
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "claims": [
     {
       "claim": "Fine-tuning Code-Llama-7B dramatically increases syntax correctness from 54.85% to 88.44% and functionality from 0% to 53.20%",
       "evidence": "Figure 6(a) and Section 5.2 show pass@3 results comparing non-finetuned vs finetuned models on the MachineGen test set.",
       "supported": "moderate"
     },
     {
       "claim": "Chain-of-thought prompting improves syntax correctness from 88.44% to 94.33% and functionality from 53.20% to 61.45%",
       "evidence": "Figure 6(b) and Section 5.3 compare results with and without CoT on the finetuned model.",
       "supported": "moderate"
     },
     {
       "claim": "Two-step feedback loops significantly improve syntax and functionality, with diminishing returns after the first iteration",
       "evidence": "Figures 7 and 8 in Section 5.4 show pass@3 results across 0, 1, and 2 feedback loop iterations for both syntax and functionality feedback.",
       "supported": "moderate"
     },
     {
       "claim": "HLS-based designs require approximately 3–4x fewer tokens than equivalent HDL implementations",
       "evidence": "Figure 2 shows a token comparison between HLS and Verilog for an identical 16-bit multiplier design, with a normalized bar chart.",
       "supported": "weak"
     },
     {
       "claim": "Performance degrades with increasing code complexity (easy: 63.33% → difficult: 53.33% functionality)",
       "evidence": "Table 2 in Section 5.6 shows functionality declining from 63.33% (easy) to 53.33% (difficult), while syntax drops from 96.67% to 90%.",
       "supported": "weak"
     },
     {
       "claim": "The model performs significantly worse on human-refined prompts (47.29% syntax, 21.36% functionality) compared to machine-generated prompts (93.83%, 62.24%)",
       "evidence": "Table 3 in Section 5.7 compares MachineGen and HumanRefine test sets.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Fine-tuning Code-Llama-7B on a collected HLS dataset of 42,000+ programs dramatically improves syntax correctness (54.85%→88.44%) and functionality (0%→53.20%) compared to the base model. Chain-of-thought prompting and iterative feedback loops provide further incremental gains. However, performance drops sharply on human-refined prompts (syntax: 93.83%→47.29%), suggesting the model overfits to machine-generated prompt patterns and may not generalize to real user instructions.",
   "red_flags": [
     {
       "flag": "Very small and non-diverse test set",
       "detail": "Only 52 base designs from two repositories, split 4:1, yielding ~10 test designs. The 42,000+ figure comes from pragma combinations of these same 52 designs, meaning extremely limited design diversity. Claims about 'automated hardware generation' rest on a tiny sample of hardware types."
     },
     {
       "flag": "No external baselines",
       "detail": "All comparisons are between ablations of the authors' own system. Despite discussing VeriGen, VerilogEval, RTLFixer, and LLM-VeriPPA in the related work, none are compared against. It is impossible to assess whether this approach is competitive with existing methods."
     },
     {
       "flag": "Unaddressed contamination risk",
       "detail": "Benchmark HLS designs are sourced from public GitHub repositories (HLSyn, ML4Accel). Code-Llama was pre-trained on GitHub data. The non-finetuned model's 54.85% syntax rate could partly reflect pre-training memorization. This is never discussed."
     },
     {
       "flag": "Train-test non-independence",
       "detail": "Training and test sets are derived from the same 52 base designs with different pragma configurations. A model could learn the base design patterns from training and generalize to test by recognizing the same structure with different pragmas, inflating apparent generalization."
     },
     {
       "flag": "No error bars or repeated trials",
       "detail": "All results are single-run point estimates without variance, confidence intervals, or repeated trials. Given the small test set size, random variation could substantially affect reported percentages."
     },
     {
       "flag": "MachineGen/HumanRefine gap undermines practical claims",
       "detail": "The dramatic performance collapse on human-refined prompts (93.83%→47.29% syntax) suggests the model is pattern-matching machine-generated prompt structure rather than understanding hardware design. This severely limits practical applicability but is not framed as a central finding."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and Codex model — foundational work for LLM code generation evaluation that this paper's methodology builds upon."
     },
     {
       "title": "VerilogEval: Evaluating large language models for Verilog code generation",
       "authors": ["Mingjie Liu", "Nathaniel Pinckney", "Brucek Khailany", "Haoxing Ren"],
       "year": 2023,
       "relevance": "LLM benchmark for hardware description language generation that directly motivates this paper's HLS alternative approach."
     },
     {
       "title": "RTLLM: An open-source benchmark for design RTL generation with large language model",
       "authors": ["Yao Lu", "Shang Liu", "Qijun Zhang", "Zhiyao Xie"],
       "year": 2024,
       "relevance": "Open-source RTL generation benchmark used for comparison of data availability between HDL and software language datasets."
     },
     {
       "title": "Verigen: A large language model for Verilog code generation",
       "authors": ["Shailja Thakur", "Baleegh Ahmad", "Hammond Pearce"],
       "year": 2023,
       "relevance": "LLM fine-tuned for Verilog generation — a direct comparison point for LLM-assisted hardware design approaches."
     },
     {
       "title": "RTLFixer: Automatically fixing RTL syntax errors with large language models",
       "authors": ["YunDa Tsai", "Mingjie Liu", "Haoxing Ren"],
       "year": 2023,
       "arxiv_id": "2311.16543",
       "relevance": "Automated LLM-assisted debugging framework for RTL designs using RAG and ReAct prompting — related feedback-loop approach for hardware code."
     },
     {
       "title": "Starcoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Large-scale code dataset and model showing HDL data scarcity relative to software languages — motivates the HLS approach."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "The base model (Code-Llama-7B) fine-tuned in this paper's experiments."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational CoT prompting technique applied and evaluated in this paper for HLS code generation."
     },
     {
       "title": "RLTF: Reinforcement learning from unit test feedback",
       "authors": ["Jiate Liu", "Yiqin Zhu", "Kaiwen Xiao"],
       "year": 2023,
       "arxiv_id": "2307.04349",
       "relevance": "Uses unit test feedback for code generation improvement — related feedback loop methodology applied to software code."
     },
     {
       "title": "LLM-VeriPPA: Power, Performance, and Area-aware Verilog Code Generation and Refinement with Large Language Models",
       "authors": ["Anonymous Authors"],
       "year": 2024,
       "relevance": "Two-stage RTL refinement process for syntax, functionality, and hardware performance — closest related work for iterative LLM hardware generation."
     },
     {
       "title": "Structured chain-of-thought prompting for code generation",
       "authors": ["Jia Li", "Ge Li", "Yongmin Li", "Zhi Jin"],
       "year": 2023,
       "arxiv_id": "2305.06599",
       "relevance": "Structured CoT for code generation showing specialized prompting needed for software code — contrasted with this paper's finding that simple CoT works for HLS."
     },
     {
       "title": "Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning",
       "authors": ["Daya Guo"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Advanced reasoning model mentioned as unavailable during this research — acknowledged limitation for future evaluation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "HLS code generation is a niche hardware engineering application; the framework is not released, limiting immediate practitioner use."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The HLS-over-HDL argument for LLM generation is reasonable but not surprising; the MachineGen/HumanRefine gap is the most unexpected finding."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk implications are raised by LLM-assisted HLS code generation."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversial claims or challenges to existing work; a straightforward empirical study."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, data, or demo released; the framework cannot be tried."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Cambridge and Imperial College London are well-known universities but not prominent AI labs in the LLM space."
     }
   }
 }