ai-research-survey

scan.json (29853B)

---

 {
   "paper": {
     "title": "HAVEN: Hallucination-Mitigated LLM for Verilog Code Generation Aligned with HDL Engineers",
     "authors": [
       "Yiyao Yang",
       "Fu Teng",
       "Pengju Liu",
       "Mengnan Qi",
       "Chenyang Lv",
       "Ji Li",
       "Xuhong Zhang",
       "Zhezhi He"
     ],
     "year": 2025,
     "venue": "Design, Automation and Test in Europe",
     "arxiv_id": "2501.04908",
     "doi": "10.23919/DATE64628.2025.10993072"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "HAVEN introduces a hallucination taxonomy for Verilog code generation (symbolic, knowledge, logical) and addresses each type through a three-stage methodology: symbolic interpretation chain-of-thought (SI-CoT), knowledge-enhanced dataset, and logical-enhanced dataset. Fine-tuned on ~62K data samples across three base models (CodeLlama-7B, DeepSeek-Coder-6.7B, CodeQwen-7B), HAVEN achieves up to 61.1% pass@1 on VerilogEval-Human, outperforming OriGen by 6.7 percentage points. On symbolic modality tasks specifically, HAVEN-CodeQwen achieves 47.4% pass@1, substantially outperforming GPT-4 (22.7%) and DeepSeek-Coder-V2 (34.1%).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'HAVEN is publicly available at https://github.com/Intelligent-Computing-Research-Group/HaVen' with a direct GitHub URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation benchmarks (VerilogEval v1, RTLLM v1.1, VerilogEval v2) are publicly available. The training data pipeline is described but the synthesized KL-dataset release is not explicitly confirmed in the paper — however, the evaluation data is standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'two Nvidia A100-80GB GPUs' and some training hyperparameters but provides no requirements.txt, Dockerfile, or detailed library/framework versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub repository is referenced but the paper itself contains no reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables IV, V, and Figures 3-4 report only point estimates (e.g., '61.1%') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'outperforms' and 'significantly improves' based solely on comparing point estimates in Tables IV and V. No statistical significance tests (p-values, t-tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., 'up to a 6.7% increase in pass@1' over OriGen, and '4.7% increase in pass@5' (Section IV-B). Absolute values are provided for both methods."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why n=10 trials for pass@k, or why the specific benchmark sizes (143/156/29 tasks) are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. The pass@k formula uses n=10 but only point estimates are shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table IV includes extensive baselines: general LLMs (GPT-3.5, GPT-4, StarCoder, CodeLlama, DeepSeek-Coder, CodeQwen) and Verilog-specific models (ChipNeMo, RTLCoder, BetterV, AutoVCoder, OriGen)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include OriGen (2024), AutoVCoder (2024), BetterV (2024), DeepSeek-Coder-V2 (2024), and RTLCoder (2024), which represent the contemporary state of the art in Verilog code generation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows a five-setting ablation (base, vanilla, vanilla+CoT, vanilla+KL, vanilla+CoT+KL) across all three base models. Figure 4 shows the effect of K-dataset and L-dataset composition independently."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports pass@1 and pass@5 for functional correctness, plus syntax correctness (syntax pass@5) on RTLLM (Table IV)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated through pass@k on benchmark test suites. No human evaluation of generated Verilog code quality is included, despite claims of alignment with HDL engineer practices."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses established external benchmarks (VerilogEval v1, RTLLM v1.1, VerilogEval v2) that are separate from the fine-tuning data (synthesized KL-dataset from GitHub code)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table V breaks down performance by symbolic modality type (truth table, waveform, state diagram). Table IV breaks down by benchmark. Figure 4 analyzes K-dataset vs L-dataset contributions separately."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Table II shows hallucination examples as motivation but these are not failure cases of HAVEN itself. No discussion of where HAVEN fails or error analysis of HAVEN's incorrect outputs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that CodeLlama 'performs worse than the other two models' after fine-tuning (Section IV-B), and acknowledges HAVEN-DeepSeek's syntax pass@5 is '4-6% lower' than AutoVCoder despite using a much smaller dataset."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'significantly improves the correctness of Verilog code generation, outperforming state-of-the-art LLM-based Verilog generation methods on VerilogEval and RTLLM benchmark.' Table IV supports these claims with HAVEN variants achieving top results across all three benchmarks."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims that SI-CoT and KL-dataset 'improve' performance are supported by the ablation study (Fig. 3), which adds components incrementally in a controlled manner. The ablation design is adequate for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims alignment 'with HDL Engineers' but evaluation is limited to automated benchmarks. Claims about 'practices of HDL engineers' (Section I) extend beyond what pass@k on benchmarks can demonstrate. No actual HDL engineers evaluated the outputs."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. The paper does not consider confounds such as the effect of additional training data volume, possible benchmark-specific overfitting, or whether improvements come from the framework vs. the data augmentation alone."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures pass@k on benchmarks but frames results as 'aligned with practices of HDL engineers' and 'suitable for the use cases of prompts generated by engineers.' No discussion of the gap between benchmark pass@k and actual HDL engineering utility."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Base models are specified as 'CodeLlama-7b-Instruct', 'Deepseek-Coder-6.7b-Instruct', and 'CodeQwen1.5-7B-Chat' which are reasonably specific. However, GPT-3.5 used for data synthesis (a critical pipeline component) has no version specified, and GPT-4 used as a baseline has no version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes what SI-CoT does and shows example interpretations (Table III) but does not provide the actual prompt text used for the CoT prompting model or the code generation prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A reports: AdamW optimizer, cosine learning rate scheduler, 15 warm-up iterations, learning rate 5e-5, global batch size 256, 3 epochs, temperature settings of 0.2/0.5/0.8, and n=10 for pass@k."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "HAVEN uses a two-stage pipeline (CoT prompting model → CodeGen-LLM) but this is not agentic scaffolding — there are no tools, loops, retry logic, or memory management. It is a straightforward inference pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C/D documents the full pipeline: ~550K Verilog samples collected from GitHub → GPT-3.5 generates vanilla instructions → parser (slang) for topic matching → exemplar-based augmentation → Verilog compiler verification → ~43K vanilla pairs, 14K K-dataset, 5K L-dataset."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper moves directly from experiments (Section IV) to conclusion (Section V) without discussing limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what the results do NOT show, what settings were not tested, or what limitations apply to the generalization of results."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation benchmarks are public, but the synthesized training datasets (43K vanilla pairs, 14K K-dataset, 5K L-dataset) and the curated exemplars are not explicitly stated as released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C describes collecting ~550,000 Verilog code samples from public GitHub repositories, curating exemplars from textbooks, and the full data synthesis pipeline including parser-based topic matching and GPT-3.5 augmentation."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are public GitHub repositories and standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented with counts at each stage: ~550K raw samples → vanilla instruction generation → parser matching → augmentation → compiler verification → yielding ~43K vanilla pairs, 14K K-dataset, and 5K L-dataset (Sections III-C and III-D)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed: 'This work is partially supported by National Key R&D Program of China (2022YFB4500200), National Natural Science Foundation of China (No.62102257).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Shanghai Jiao Tong University, Zhejiang University, and one independent researcher. No commercial product affiliation conflict exists."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from Chinese government research programs (National Key R&D Program, NSFC), which have no financial stake in the specific experimental outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff dates for the base models (CodeLlama, DeepSeek-Coder, CodeQwen) are not stated. The paper does not discuss when their training data ends."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether VerilogEval or RTLLM benchmark problems appeared in the base models' pre-training data or in the fine-tuning data collected from GitHub."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "VerilogEval was published in 2023 and the base models were likely trained on data that could include it. No contamination analysis is performed or discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Evaluation is entirely automated benchmark-based."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed per example are reported. The two-stage pipeline (CoT + CodeGen) presumably doubles inference cost but this is not discussed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'two Nvidia A100-80GB GPUs' and '3 epochs' but does not quantify total training time, GPU hours, or computational cost."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed experiments. Results are reported as single point estimates without seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states n=10 trials per problem for the pass@k metric (Section IV-A, Equation 1)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper tries temperatures 0.2, 0.5, and 0.8 and reports 'the best performance,' but does not disclose the full search budget or which temperature was selected for each model/benchmark combination."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we set the temperature of each model to 0.2, 0.5 and 0.8, reporting the best performance' (Section IV-A). This reports the maximum across three configurations without specifying which was best or using a validation set for selection."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made across models, benchmarks, and ablation settings. No statistical tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their HAVEN system against baselines without acknowledging the bias of evaluating one's own system. Several baselines (BetterV, AutoVCoder, ChipNeMo) are not publicly available, so results may come from the original papers under different conditions."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of compute differences. HAVEN uses a two-stage pipeline (CoT + CodeGen) which presumably requires more compute than single-stage baselines, but this is not compared or discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses VerilogEval v2 because it 'focuses on human evaluation problems' but does not analyze whether pass@k on any benchmark actually measures alignment with HDL engineer practices as claimed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HAVEN adds a CoT pre-processing stage that other baselines lack, creating a scaffold confound. When comparing HAVEN (CoT + fine-tuned model) against baselines (single-stage models), the improvement could be from the scaffold rather than the model. This is partially addressed in the ablation but not when comparing against external baselines."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "VerilogEval v1 was published in 2023. The base models could have been trained on data including its solutions. No temporal leakage analysis is provided."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The SI-CoT interpretation step provides additional context that baselines don't receive, which could constitute a form of feature advantage rather than leakage, but this asymmetry is not analyzed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The fine-tuning data comes from ~550K GitHub Verilog samples. No analysis of whether these overlap with VerilogEval or RTLLM benchmark sources."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used (no canary strings, membership inference, n-gram overlap, or decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "HAVEN-CodeQwen outperforms OriGen by 6.7% in pass@1 and 4.7% in pass@5 on VerilogEval(v1)-Human.",
       "evidence": "Table IV shows HAVEN-CodeQwen achieves 61.1% pass@1 and 64.8% pass@5 vs. OriGen's 54.4% pass@1 and 60.1% pass@5 on VerilogEval-Human (Section IV-B).",
       "supported": "moderate"
     },
     {
       "claim": "HAVEN-DeepSeek achieves the best functional pass@5 on RTLLM v1.1, outperforming both OriGen and GPT-4 by 0.5%.",
       "evidence": "Table IV shows HAVEN-DeepSeek at 66.0% functional pass@5 on RTLLM v1.1, vs. OriGen at 65.5% and GPT-4 at 65.5%.",
       "supported": "weak"
     },
     {
       "claim": "SI-CoT improves pass@1 and pass@5 by 3.6% and 6.6% on average across three base models.",
       "evidence": "Figure 3 ablation study comparing 'vanilla' vs. 'vanilla+CoT' settings on VerilogEval(v1)-Human across CodeLlama, DeepSeek-Coder, and CodeQwen (Section IV-D).",
       "supported": "moderate"
     },
     {
       "claim": "KL-dataset fine-tuning improves pass@1 and pass@5 by 12.3% and 8.7% on average.",
       "evidence": "Figure 3 ablation comparing 'vanilla' vs. 'vanilla+KL' settings across three base models on VerilogEval(v1)-Human (Section IV-D).",
       "supported": "moderate"
     },
     {
       "claim": "HAVEN-CodeQwen achieves the highest overall pass@1 of 47.4% on symbolic modality tasks, substantially outperforming GPT-4 (22.7%) and DeepSeek-Coder-V2 (34.1%).",
       "evidence": "Table V shows performance on 44 curated symbolic modality tasks from VerilogEval(v1)-Human with breakdown by truth table, waveform, and state diagram (Section IV-C).",
       "supported": "moderate"
     },
     {
       "claim": "SI-CoT helps commercial LLMs improve on symbolic modality tasks without fine-tuning.",
       "evidence": "Table VI shows GPT-4o mini improves from 22.7% to 31.8%, GPT-4 from 22.7% to 34.1%, and DeepSeek-Coder-V2 from 34.1% to 45.5% with SI-CoT (Section IV-D).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Best-temperature cherry-picking",
       "detail": "The paper tries 3 temperature settings (0.2, 0.5, 0.8) and reports 'the best performance' without disclosing which temperature was selected per model/benchmark or using a validation set for selection. This inflates reported numbers relative to what a practitioner would achieve."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All 'outperforms' and 'significantly improves' claims rest on comparing point estimates without significance tests. Several claimed improvements are small (e.g., 0.5% on RTLLM functional pass@5) and could easily be within random variation."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats to validity, or scope boundary discussion despite making broad claims about alignment with HDL engineer practices based solely on automated benchmarks."
     },
     {
       "flag": "Non-reproducible baselines",
       "detail": "Several baselines (ChipNeMo, BetterV, AutoVCoder) are noted as 'not publicly accessible at the time of writing,' making independent verification of comparative claims impossible."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "VerilogEval was published in 2023 and the fine-tuning data comes from GitHub repositories. No analysis of whether benchmark problems or solutions appear in the training data."
     }
   ],
   "cited_papers": [
     {
       "title": "CodeHalu: Investigating Code Hallucinations in LLMs via Execution-based Verification",
       "authors": ["Y. Tian", "W. Yan", "Q. Yang"],
       "year": 2024,
       "arxiv_id": "2405.00253",
       "relevance": "Proposes a hallucination taxonomy for LLM code generation, directly informing HAVEN's hallucination classification approach."
     },
     {
       "title": "Exploring and Evaluating Hallucinations in LLM-Powered Code Generation",
       "authors": ["F. Liu", "Y. Liu", "L. Shi"],
       "year": 2024,
       "arxiv_id": "2404.00971",
       "relevance": "Analyzes hallucination phenomena in LLM-based code generation (HalluCode), providing context for hallucination mitigation work."
     },
     {
       "title": "RTLFixer: Automatically Fixing RTL Syntax Errors with Large Language Model",
       "authors": ["Y. Tsai", "M. Liu", "H. Ren"],
       "year": 2024,
       "doi": "10.1145/3649329.3657353",
       "relevance": "Addresses syntax error fixing in LLM-generated RTL code, a related approach to improving hardware code generation quality."
     },
     {
       "title": "OriGen: Enhancing RTL Code Generation with Code-to-Code Augmentation and Self-Reflection",
       "authors": ["F. Cui", "C. Yin", "K. Zhou"],
       "year": 2024,
       "relevance": "State-of-the-art LLM-based Verilog code generation method using code augmentation, serving as HAVEN's primary baseline."
     },
     {
       "title": "RTLCoder: Outperforming GPT-3.5 in Design RTL Generation with Our Open-Source Dataset and Lightweight Solution",
       "authors": ["S. Liu", "W. Fang", "Y. Lu"],
       "year": 2024,
       "relevance": "Open-source RTL generation framework with dataset, baseline and methodological precursor for HAVEN's data augmentation approach."
     },
     {
       "title": "AutoVCoder: A Systematic Framework for Automated Verilog Code Generation Using LLMs",
       "authors": ["M. Gao", "J. Zhao", "Z. Lin"],
       "year": 2024,
       "relevance": "Systematic Verilog code generation framework fine-tuned on ~1M samples, key baseline for HAVEN's data efficiency claims."
     },
     {
       "title": "VerilogEval: Evaluating Large Language Models for Verilog Code Generation",
       "authors": ["M. Liu", "N. Pinckney", "B. Khailany", "H. Ren"],
       "year": 2023,
       "relevance": "Primary benchmark used for evaluating LLM-based Verilog generation capability, including machine and human task splits."
     },
     {
       "title": "Revisiting VerilogEval: Newer LLMs, In-Context Learning, and Specification-to-RTL Tasks",
       "authors": ["N. Pinckney", "C. Batten", "M. Liu"],
       "year": 2024,
       "arxiv_id": "2408.11053",
       "relevance": "VerilogEval v2 benchmark extending evaluation to specification-to-RTL design tasks more aligned with HDL engineer practices."
     },
     {
       "title": "RTLLM: An Open-Source Benchmark for Design RTL Generation with Large Language Model",
       "authors": ["Y. Lu", "S. Liu", "Q. Zhang", "Z. Xie"],
       "year": 2024,
       "doi": "10.1109/ASP-DAC58780.2024.10473904",
       "relevance": "RTL-focused benchmark evaluating design generation with LLMs, used as one of three evaluation benchmarks."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational paper on evaluating code generation LLMs, introducing the pass@k metric used throughout HAVEN's evaluation."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "authors": ["D. Guo"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Code-specific LLM used as one of HAVEN's base models for fine-tuning."
     },
     {
       "title": "BetterV: Controlled Verilog Generation with Discriminative Guidance",
       "authors": ["Z. Pei", "H. Zhen", "M. Yuan"],
       "year": 2024,
       "relevance": "Controlled Verilog generation approach using discriminative guidance, baseline comparison for HAVEN."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2024,
       "relevance": "Examines rigor of LLM code generation evaluation, relevant to survey scope of code generation methodology."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Open-source framework with code released for Verilog code generation — useful for HDL engineers and chip design teams, though the domain is niche."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The hallucination taxonomy for Verilog is a novel framing, but the core approach (CoT + data augmentation + fine-tuning) follows established patterns."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI risk or safety concerns raised; focused on improving code generation correctness."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy; standard academic contribution in the LLM-for-hardware space."
     },
     "demo_ability": {
       "score": 2,
       "justification": "GitHub repository is provided and models are open-source at 7B parameters, potentially runnable on consumer hardware."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From Shanghai Jiao Tong University and Zhejiang University — respected institutions but not high-profile AI labs in the public consciousness."
     }
   }
 }