ai-research-survey

scan.json (28085B)

---

 {
   "paper": {
     "title": "Toward Hardware Security Benchmarking of LLMs",
     "authors": [
       "Raheel Afsharmazayejani",
       "Mohammad Moradi Shahmiri",
       "Parker Link",
       "Hammond Pearce",
       "Benjamin Tan"
     ],
     "year": 2024,
     "venue": "IEEE LLM Aided Design Workshop (LAD)",
     "doi": "10.1109/LAD62341.2024.10691745"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "This paper proposes a security-focused evaluation suite for LLM-generated HDL code, with preliminary tests on GPT-3.5 and CodeLlama across 7 benchmark scenarios. GPT-3.5 produces syntactically correct code (>80% syntax pass rate) but security-correct code at much lower rates, with increased prompt verbosity generally helping and increased code redaction hurting FPV pass rates. CodeLlama performed substantially worse, with most responses failing to compile (~43% compilation failure, ~40% non-code responses). The work highlights a critical gap in security-centric evaluation of LLM-generated hardware designs.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark suite is released at https://github.com/CalgaryISH/llm-hwsec-benchmarking, explicitly stated in Section I: 'we make our benchmark suite available at [URL] for issues, discussions, and pull requests.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark suite including test scenarios, prompts, and security assertions is released at the GitHub repository. The benchmark designs sourced from TrustHub are publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or software version requirements are provided. They mention using Cadence JasperGold for FPV but provide no setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. The GitHub release is mentioned for contributions but no README or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw counts out of 5 runs (Tables IV-X) and percentages (Figs. 3-5) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares GPT-3.5 vs CodeLlama and different verbosity levels without any statistical tests. Claims like 'CodeLlama's overall performance is worse than GPT3.5' rest on raw number comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw pass counts out of 5 and percentages are reported. No effect sizes, relative improvements, or contextualized magnitude measures are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Each test is run 5 times (375 experiments per model) but no justification for why 5 runs was chosen. No power analysis or discussion of whether 5 runs is sufficient to detect meaningful differences."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results show individual pass counts out of 5 in tables but no variance, standard deviation, or spread measures across runs are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two LLMs are compared (GPT-3.5 and CodeLlama), and Table I compares their evaluation suite against prior work. The models serve as mutual baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "GPT-3.5 was already superseded by GPT-4 when this paper was written (2024). CodeLlama is based on Llama2, not Llama3. Neither model represents the state of the art for the evaluation year."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "While verbosity levels and code redaction lengths are varied (Figs. 3-5), these are parameter sweeps of the benchmark, not systematic ablations removing individual components of a proposed system."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two distinct metrics are used: syntax check pass rate (Fig. 3) and Formal Property Verification (FPV) pass rate (Fig. 4). Response type classification is also reported for CodeLlama."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A states: 'we assessed functional correctness using plausibility as a proxy; a human with domain knowledge reads the code and judges whether it would plausibly provide the intended function.' Human evaluation of LLM outputs was performed, though acknowledged as limited."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No separation of dev and test sets is described. All benchmark scenarios are used for the evaluation without any held-out partition."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Detailed per-benchmark breakdowns are provided: individual results for all 7 scenarios (REM021-RMI041) in Figs. 3-5 and Tables IV-X, broken down by verbosity level and code block size."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "CodeLlama's compilation failures are extensively analyzed (~43% compilation failure, ~40% non-code, ~3% incomplete). GPT-3.5's declining FPV with increased redaction is discussed. Section V discusses multiple failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "CodeLlama's near-complete inability to produce compilable HDL is a significant negative result. Low-verbosity prompts never achieving highest FPV pass rates is reported. Medium verbosity sometimes performing worst (REM041) is noted."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are modest: 'proposes an approach for evaluating the security,' 'new insights into the challenges,' and 'illustrative preliminary use.' All are supported by the paper's content in Sections III-V."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are modest and based on controlled manipulation. 'Increased redaction results in decreased FPV' (Section IV) comes from systematically varying a single parameter (code block size) while holding others constant, which is adequate for the limited claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently uses hedging language: 'preliminary,' 'illustrative,' 'initial foray,' 'initial set of test scenarios.' The title itself says 'Toward.' Claims are bounded to the two tested LLMs and the specific benchmark subset."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V discusses data contamination as an alternative explanation, notes that passing assertions doesn't prove functional correctness, and acknowledges that prompt wording could explain results (Section V-B 'Diverse prompting')."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B 'Assertions' explicitly acknowledges the proxy gap: 'passing an assertion does not always prove that the designer's original intentions have been honoured' and discusses how assertions target individual vulnerabilities without guaranteeing overall security."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT3.5' and 'CodeLlama' without specific version identifiers. No snapshot date, API version, or model size is specified for either model. CodeLlama is described only as 'a fine-tuned version of Llama2.'"
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Actual prompt text is provided for all three verbosity levels in Figs. 2 and 6. A code snippet example is shown in Fig. 7. The benchmark suite with full prompts is released on GitHub."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "For GPT-3.5: 'the default temperature for the web interface was used' without stating the actual value. For CodeLlama: 'temperature of 0.8.' Other parameters (top-p, max tokens) are not mentioned for either model."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. LLMs are prompted directly with code completion tasks."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III describes how benchmark scenarios were constructed: designs sourced from TrustHub and CWEs, code blocks designated for removal at varying lengths, three verbosity levels created per scenario. The process from design selection to prompt construction is documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B is titled 'Limitations and Opportunities' with substantial discussion covering limited sample size, automation bottlenecks, diverse prompting, test data contamination, assertion limitations, and security evaluation scope."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: data contamination from publicly available TrustHub circuits, limited sample size constraining granularity, manual post-processing bottleneck, and that assertion passing doesn't guarantee functional correctness. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Clear scope boundaries: only two LLMs evaluated, only TrustHub-sourced benchmarks used in preliminary results, evaluation limited to FPV-based security assessment, functional correctness only manually checked. Section V explicitly lists what remains as open challenges."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark suite is released but the actual raw LLM responses from the 750 experiments (375 per model) are not made available. Only aggregated pass counts (Tables IV-X) are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV describes the collection process: each scenario queried at 3 verbosity levels across multiple code block variants, each run 5 times, using GPT-3.5's web interface and CodeLlama at temperature 0.8. FPV performed using Cadence JasperGold."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Benchmark scenarios were author-constructed from existing security benchmarks (TrustHub, CWEs), not sampled from a population."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline steps are described (prompt → LLM response → syntax check → FPV) but exact counts at each filtering stage are not consistently reported. For GPT-3.5, it is unclear how many designs passed syntax check before FPV evaluation. CodeLlama's filtering is better documented (~43% compilation failure, ~40% non-code, ~3% incomplete, ~14% compiled)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed: 'supported in part by the Natural Sciences and Engineering Research Council of Canada (NSERC) [RGPIN-2022-03027]' and 'partly supported by a gift from Intel Corporation.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: University of Calgary (Canada) and University of New South Wales (Australia). Authors are not affiliated with OpenAI or Meta (makers of the evaluated LLMs)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSERC is a government funding agency with no stake in LLM performance. Intel's gift is disclosed with the note 'This work does not in any way constitute an Intel endorsement of a product/supplier.' Neither funder has a direct interest in the outcome of GPT-3.5 or CodeLlama evaluations."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. The Intel gift is disclosed but there is no explicit declaration of whether authors hold patents or equity related to the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for either GPT-3.5 or CodeLlama. Without cutoff dates, it is impossible to assess whether TrustHub benchmarks were in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section II point 3 explicitly states 'LLMs are trained on solutions: Problem sets like HDLBits are freely available; thus will have been used in the training of many LLMs evaluated.' Section V-B also discusses contamination risk for their benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is discussed as a specific limitation in Section V-B: 'it is safe to assume that any representative, real-world benchmark is at risk for cross-contamination between training and test data, including the current work.' Also flagged as a desideratum in Section II."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is entirely automated (LLM prompting, syntax checking, FPV)."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLM-generated code against automated security assertions."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, or per-experiment cost is reported for either GPT-3.5 or CodeLlama despite running 750 total experiments."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. No mention of GPU hours for CodeLlama, API costs for GPT-3.5, or JasperGold FPV compute time."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Each test is run 5 times but results are reported as raw pass counts, not as mean/std across runs. No explicit seed sensitivity analysis is conducted."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C (Table III) clearly states: 'Each scenario is repeated five times, adding up to 375 experiments run on each evaluated model.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search was conducted. Temperature was set to default for GPT-3.5 and 0.8 for CodeLlama without justification or exploration of other values."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of temperature 0.8 for CodeLlama is not justified. GPT-3.5's 'default temperature' is used without explanation. No validation set or selection criterion is described for these choices."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the benchmark, crafted the assertions, and evaluated the LLMs without acknowledging potential bias in their assertion design or evaluation process."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "GPT-3.5 (commercial API) and CodeLlama (open-source, locally run) have very different compute profiles, but no discussion of compute budget differences is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B 'Assertions' discusses construct validity: 'passing an assertion does not always prove that the designer's original intentions have been honoured' and notes that security assertions target individual vulnerabilities without guaranteeing overall design security."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. LLMs are prompted directly without any agentic framework or tooling."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While contamination is discussed generally, no specific temporal analysis is performed. The paper does not state when TrustHub benchmarks were published relative to GPT-3.5 or CodeLlama training cutoffs."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The medium and high verbosity prompts explicitly describe the vulnerability to be fixed, effectively providing the answer. This information leakage is designed as an experimental variable but not discussed as a potential confound for security evaluation claims."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark scenarios (sourced from TrustHub and CWEs) share structural similarities with each other or with LLM training data examples."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. Contamination is only discussed as a conceptual concern without any detection analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-3.5 produces syntactically correct HDL code in most cases, with >80% syntax check pass rate for most benchmarks",
       "evidence": "Fig. 3 shows syntax check rates across 7 benchmark scenarios at 3 verbosity levels, with most exceeding 80% (Section IV)",
       "supported": "moderate"
     },
     {
       "claim": "Increased code redaction generally leads to decreased FPV (security) pass rates for GPT-3.5",
       "evidence": "Fig. 5 shows FPV pass rates by number of removed lines. 'Except for REM021 and RMI041, increased redaction results in decreased FPV' (Section IV)",
       "supported": "moderate"
     },
     {
       "claim": "Low-verbosity prompts never achieve the highest FPV pass rate across any benchmark",
       "evidence": "Section IV: 'In none of the benchmarks a low-verbosity prompt provides the highest FPV pass rate.' Supported by Fig. 4 data across all 7 benchmarks.",
       "supported": "moderate"
     },
     {
       "claim": "CodeLlama's overall performance is substantially worse than GPT-3.5 on the evaluation suite",
       "evidence": "Section IV: Out of 375 CodeLlama experiments, ~43% compilation failure, ~40% non-code responses, ~3% incomplete, ~14% compiled. Nearly all compiled samples failed to produce functionally correct code.",
       "supported": "strong"
     },
     {
       "claim": "Security-centric evaluation of LLM-generated HDL code is a significant gap in the literature",
       "evidence": "Table I shows only 4 prior works with security-related evaluations, most with small test sets. Section II catalogs 5 specific weaknesses in existing evaluations.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Very small preliminary evaluation",
       "detail": "Only 2 LLMs evaluated on 7 benchmark scenarios with 5 runs each. One of the two models (CodeLlama) produced almost no usable results, leaving effectively a single-model evaluation."
     },
     {
       "flag": "No statistical analysis",
       "detail": "All comparisons are based on raw counts out of 5 without any statistical tests, confidence intervals, or variance reporting. With n=5 per condition, differences between conditions may be entirely due to sampling noise."
     },
     {
       "flag": "Outdated model selection",
       "detail": "GPT-3.5 was already superseded by GPT-4 at publication time (2024). CodeLlama is based on Llama2 rather than more recent models. Results may not reflect current LLM capabilities."
     },
     {
       "flag": "Functional correctness not rigorously verified",
       "detail": "Functional correctness was assessed by human 'plausibility' judgment, which the authors themselves acknowledge is subjective and unscalable. Security assertions can pass on code that is not functionally correct."
     },
     {
       "flag": "Assertion quality varies by design",
       "detail": "The authors acknowledge 'the availability and quality of the assertions in our work vary considerably, making it difficult to do a balanced comparison' (Section V-A). This undermines cross-benchmark comparisons."
     }
   ],
   "cited_papers": [
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "authors": ["H. Pearce", "B. Ahmad", "B. Tan", "B. Dolan-Gavitt", "R. Karri"],
       "year": 2022,
       "relevance": "Foundational study on security vulnerabilities in LLM-generated code, finding 40% of generated code contains vulnerabilities."
     },
     {
       "title": "VeriGen: A Large Language Model for Verilog Code Generation",
       "authors": ["S. Thakur", "B. Ahmad", "H. Pearce", "B. Tan", "B. Dolan-Gavitt", "R. Karri", "S. Garg"],
       "year": 2023,
       "arxiv_id": "2308.00708",
       "relevance": "LLM benchmark for Verilog code generation using HDLBits as foundation; demonstrates LLM capability and limitations for HDL."
     },
     {
       "title": "VerilogEval: Evaluating Large Language Models for Verilog Code Generation",
       "authors": ["M. Liu", "N. Pinckney", "B. Khailany", "H. Ren"],
       "year": 2023,
       "arxiv_id": "2309.07544",
       "relevance": "Benchmark evaluation of LLMs for Verilog code generation with 156 test scenarios."
     },
     {
       "title": "(Security) Assertions by Large Language Models",
       "authors": ["R. Kande", "H. Pearce", "B. Tan", "B. Dolan-Gavitt", "S. Thakur", "R. Karri", "J. Rajendran"],
       "year": 2024,
       "relevance": "Evaluates LLM capability in generating hardware security assertions, directly related to security evaluation of LLM-generated designs."
     },
     {
       "title": "On Hardware Security Bug Code Fixes By Prompting Large Language Models",
       "authors": ["B. Ahmad", "S. Thakur", "B. Tan", "R. Karri", "H. Pearce"],
       "year": 2024,
       "relevance": "Explores LLM prompting for fixing hardware security bugs, related to LLM capability in security-critical code generation."
     },
     {
       "title": "AutoChip: Automating HDL Generation Using LLM Feedback",
       "authors": ["S. Thakur", "J. Blocklove", "H. Pearce", "B. Tan", "S. Garg", "R. Karri"],
       "year": 2023,
       "arxiv_id": "2311.04887",
       "relevance": "LLM-based automated HDL generation with feedback loops, demonstrating agentic approaches to hardware design."
     },
     {
       "title": "Purple Llama CyberSecEval: A Secure Coding Benchmark for Language Models",
       "authors": ["M. Bhatt", "S. Chennabasappa", "C. Nikolaidis"],
       "year": 2023,
       "arxiv_id": "2312.04724",
       "relevance": "Security-focused benchmark for LLM-generated software code using automated insecure code detectors."
     },
     {
       "title": "All Artificial, Less Intelligence: GenAI through the Lens of Formal Verification",
       "authors": ["D. N. Gadde", "A. Kumar", "T. Nalapat", "E. Rezunov", "F. Cappellini"],
       "year": 2024,
       "arxiv_id": "2403.16750",
       "relevance": "Evaluates GenAI-generated hardware through formal verification with 30 security-related tests."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["B. Rozière", "J. Gehring", "F. Gloeckle"],
       "year": 2024,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code generation model evaluated in this study; key baseline for code LLM capability."
     },
     {
       "title": "AssertLLM: Generating and Evaluating Hardware Verification Assertions from Design Specifications via Multi-LLMs",
       "authors": ["W. Fang", "M. Li", "M. Li", "Z. Yan", "S. Liu", "H. Zhang", "Z. Xie"],
       "year": 2024,
       "arxiv_id": "2402.00386",
       "relevance": "Multi-LLM approach to generating hardware verification assertions from design specifications."
     },
     {
       "title": "Time Travel in LLMs: Tracing Data Contamination in Large Language Models",
       "authors": ["S. Golchin", "M. Surdeanu"],
       "year": 2023,
       "relevance": "Methods for detecting data contamination in LLMs, directly relevant to benchmark validity concerns."
     }
   ]
 }