ai-research-survey

scan.json (26037B)

---

 {
   "paper": {
     "title": "Large Language Model for Verilog Code Generation: Literature Review and the Road Ahead",
     "authors": [
       "Guang Yang",
       "Wei Zheng",
       "Dong Liang",
       "Peng Hu",
       "Yukui Yang",
       "Shaohang Peng",
       "Zhenghan Li",
       "Jiahui Feng",
       "Xiao Wei",
       "Kexin Sun",
       "Deyuan Ma",
       "Haotian Cheng",
       "Yiheng Shen",
       "Xiang Chen",
       "Xing Hu",
       "Terry Yue Zhuo",
       "David Lo"
     ],
     "year": 2025,
     "venue": "ACM Computing Surveys",
     "arxiv_id": "2512.00020",
     "doi": "10.48550/arXiv.2512.00020"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This systematic literature review analyzes 102 papers (70 peer-reviewed, 32 preprints) on LLM-based Verilog code generation from 2020-2025, documenting exponential growth from 1 paper in 2020 to 66 in 2025. The field has converged on code-specialized foundation models (DeepSeek-Coder, Qwen2.5-Coder, CodeLlama) with 34 instruction-tuned LLMs developed for Verilog, 19 with open weights. The review identifies a shift toward execution-based evaluation (functional-pass@k) as the dominant metric, and categorizes techniques into training-free (EDA feedback, prompt engineering) and training-based (SFT, RL) approaches, while highlighting critical gaps in security, efficiency alignment, copyright protection, and hallucination mitigation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, analysis code, or supplementary materials are released. The survey could have released its search queries, coding spreadsheets, or analysis scripts but did not."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No structured dataset of the 102 reviewed papers, extracted data, or coding results is released. The reference list provides paper titles but not the coded attributes or quality assessment scores."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided. While this is a survey without computational experiments, any analysis scripts or data processing tools used are not specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While Section 3 describes the search methodology in detail (databases, keywords, filtering stages), there are no step-by-step instructions for reproducing the exact search results or data extraction process."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative systematic literature review with descriptive counts and categorizations, not statistical aggregation requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The survey performs no statistical tests; it categorizes and describes trends qualitatively."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No effect sizes are computed; the survey reports paper counts and categorical breakdowns."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No statistical sampling requiring power analysis; the corpus size (102 papers) is determined by the systematic search process."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs are conducted; variance reporting is inapplicable to this survey format."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 explicitly compares this survey against 6 related literature reviews across dimensions of year, paper count, topics, and Verilog focus, positioning the contribution relative to prior work."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The compared surveys are from 2024-2025 (Jiang et al. 2024, Joel et al. 2024, Fang et al. 2025, He et al. 2024, Chen et al. 2024-2025), representing the current state of the field."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted requiring evaluation metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate; this is a literature review."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey provides detailed breakdowns by LLM type (open vs. closed source, base vs. instruction-tuned), by year, by venue (Figure 4), by dataset type (Tables 6-7), by technique category (training-free vs. training-based), and by alignment dimension (Section 7)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.1 discusses four categories of limitations: foundation/knowledge gaps, data/benchmark scarcity, evaluation/alignment deficits, and deployment readiness challenges."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies significant negative findings: no current solution integrates into industrial EDA workflows (Section 8.1), existing benchmarks are too small (<100 samples), current metrics ignore critical hardware dimensions like PPA and security, and alignment research is nascent."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to analyze 102 papers (70 published, 32 preprints) structured around 4 RQs, which is fully supported by the content. The claim of being the first comprehensive survey on LLM-based Verilog generation is supported by Table 1's comparison with existing reviews."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims of its own. It describes trends ('growth from 12 to 274 usages'), categorizes techniques, and summarizes findings from reviewed papers without asserting causal relationships from its own analysis."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The scope is clearly bounded to LLM-based Verilog code generation (not HDLs generally), covering papers from 2020 to October 2025. The title and abstract accurately reflect this specific focus."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a pure survey/taxonomy that presents descriptive categorizations rather than empirical results requiring alternative explanations."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements. It reports paper counts, categorizations, and trend descriptions, and frames them exactly as such without overreaching to broader claims about the field's overall health or impact."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The survey does not use any LLMs in its own methodology; it reviews papers that use LLMs."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in the survey's methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments requiring hyperparameters are conducted."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in the survey's methodology."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 thoroughly documents the paper selection pipeline: six databases searched with specific keyword sets (Section 3.2.1), three-stage filtering (5,172 → 687 → 124 papers) with explicit inclusion/exclusion criteria (Table 2), quality assessment with 5 QAC criteria and a 12/15 threshold (Table 3), and snowballing (186 candidates → 15 additions → 102 final)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 'Threats to Validity' provides a dedicated discussion covering three specific threats: paper search omission, study selection bias, and categorization/analysis bias."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: 'the fast-paced nature of LLM research may lead to overlooking emerging work,' 'reduction from initial candidates to the final set involved subjective quality assessments,' and 'categorizing diverse methodologies introduces potential classification bias.' Mitigations are also described for each."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The scope is explicitly bounded: Verilog code generation specifically (not all HDLs), LLM-based approaches only, publications from 2020 to October 2025, and the paper states it covers 102 papers from specific venues and databases."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No structured dataset of coded paper attributes, quality assessment scores, or extraction spreadsheets is released. Only the reference list provides partial raw data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the collection procedure in detail: manual search of 8 venues yielded 16 QGS papers, automated search across 6 databases using specified keyword sets, with temporal scope from 2020 onward."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper 'recruitment' (search strategy) is described in Section 3.2: manual search of premier venues (AAAI, ACL, ICML, ICLR, NeurIPS, DAC, TCAD), automated search across IEEE Xplore, ACM DL, ScienceDirect, Web of Science, SpringerLink, and arXiv with specified keyword combinations."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 and Section 3.3 document the full pipeline with counts at each stage: initial screening (→5,172), content-based filtering (→687), full-text assessment (→124), quality assessment (→85), snowballing (+15 → 102 final). Criteria for each stage are specified in Table 2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 11 acknowledges partial support from the National Natural Science Foundation of China (NSFC, No. 62141208)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Zhejiang University, Northwestern Polytechnical University, Nantong University, Monash University, and Singapore Management University. None appear to be affiliated with companies whose products are being reviewed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSFC is a Chinese government research funding agency with no commercial stake in the survey's conclusions about LLM-based Verilog code generation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model's capability on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational method of its own."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The paper follows the Kitchenham SLR methodology (reference [59]) with the Quasi-Gold Standard (QGS) strategy (reference [133]). Section 3 describes the full protocol: planning (RQs, search strategy), conducting (three-stage filtering, quality assessment, snowballing), and analysis, with a systematic flow diagram (Figure 3)."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides five Quality Assessment Criteria (QAC1-5) scored on a 0-3 scale with a 12/15 (80%) inclusion threshold. Criteria cover venue prestige, contribution, workflow clarity, experimental detail, and argument support. Preprints lacking venue scores must demonstrate exceptional merit in other dimensions."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss publication bias. Section 9 addresses search omission, selection bias, and categorization bias, but never considers whether the surveyed literature skews toward positive results or whether negative findings about LLM-based Verilog generation are underrepresented."
       }
     }
   },
   "claims": [
     {
       "claim": "This is the first comprehensive systematic literature review specifically focused on LLM-based Verilog code generation.",
       "evidence": "Table 1 compares against 6 related surveys, showing none provide full Verilog focus; prior surveys either cover general code generation (Jiang 2024, Joel 2024, Chen 2025) or broad EDA (Fang 2025, He 2024, Chen 2024) with only partial Verilog coverage.",
       "supported": "strong"
     },
     {
       "claim": "Research in LLM-based Verilog code generation has grown exponentially from 1 paper in 2020 to 66 by September 2025.",
       "evidence": "Figure 4b shows year-by-year counts: 1 (2020), 0 (2021), 0 (2022), 6 (2023), 29 (2024), 66 (2025). This quantitative evidence directly from the systematic search supports the exponential growth claim.",
       "supported": "strong"
     },
     {
       "claim": "Open-source Base LLM usages grew 233% from 2024 to 2025 (46 to 153 mentions), with Llama, DeepSeek, and Qwen becoming the dominant families.",
       "evidence": "Table 4 provides detailed counts by LLM family and year. Llama: 0→15→49, DeepSeek: 0→6→48, Qwen: 0→3→25 across 2023-2025.",
       "supported": "strong"
     },
     {
       "claim": "GPT series dominates closed-source usage with 149 of 179 closed-source mentions (83.2%).",
       "evidence": "Table 4 reports GPT series at 7+42+100=149 mentions vs. Claude (19), Gemini (5), and Others (6) totaling 179 closed-source mentions.",
       "supported": "strong"
     },
     {
       "claim": "34 instruction-tuned LLMs have been developed for Verilog, with 19 (55.9%) providing open weights, predominantly built on code-specialized foundations.",
       "evidence": "Table 5 lists all 34 IT LLMs with their foundation models, weight availability, and URLs. 28/34 (82.4%) use coding-oriented foundations, with DeepSeek-Coder (11 models), Qwen coder (9), and CodeLlama (8) as the top clusters.",
       "supported": "strong"
     },
     {
       "claim": "Current research has critical gaps in deployment readiness, with no solution integrating seamlessly into industrial EDA workflows.",
       "evidence": "Section 8.1(4) states this limitation, identifying lacks in interactive refinement, explainability, and human-in-the-loop features. This is a qualitative assessment synthesized from the reviewed literature.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Publication bias not addressed",
       "detail": "Despite being a systematic review, the paper never discusses whether the surveyed literature skews toward positive results about LLM-based Verilog generation. No funnel plots, no discussion of negative-result underrepresentation, and no acknowledgment that the field may systematically over-report successes. This is especially concerning for a fast-moving field with strong publication incentives."
     },
     {
       "flag": "Quality scores not reported per paper",
       "detail": "While the paper uses a 5-criterion quality assessment (QAC) for inclusion decisions, the quality scores of individual papers are not reported. This means the synthesis treats all 102 included papers as equally reliable, without weighting conclusions by the methodological strength of their sources."
     },
     {
       "flag": "No artifacts released for a survey",
       "detail": "No structured dataset, search query logs, coding spreadsheets, or analysis scripts are released. The survey's extraction and categorization cannot be independently verified or built upon, limiting its value as a living resource."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational paper on Codex for code generation, establishing the paradigm of LLM-based code synthesis that extends to hardware description languages."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major open-source code model family that is the most widely adopted foundation for Verilog-specific instruction tuning (64 mentions in the survey)."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "authors": ["Daya Guo", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Code-specialized foundation model that is the most common base for Verilog instruction-tuned LLMs (11 of 34 IT models)."
     },
     {
       "title": "VerilogEval: Evaluating Large Language Models for Verilog Code Generation",
       "authors": ["Mingjie Liu", "Nathaniel Ross Pinckney"],
       "year": 2023,
       "doi": "10.1109/ICCAD57390.2023.10323812",
       "relevance": "Key benchmark for evaluating LLM-based Verilog generation, derived from HDLBits with 156 human-crafted and 143 machine-generated problems."
     },
     {
       "title": "VeriGen: A Large Language Model for Verilog Code Generation",
       "authors": ["Shailja Thakur", "Baleegh Ahmad"],
       "year": 2024,
       "doi": "10.1145/3643681",
       "relevance": "Early open-source LLM specifically fine-tuned for Verilog generation, with released weights and training data."
     },
     {
       "title": "RTLCoder: Fully Open-Source and Efficient LLM-Assisted RTL Code Generation Technique",
       "authors": ["Shang Liu", "Wenji Fang"],
       "year": 2025,
       "doi": "10.1109/TCAD.2024.3483089",
       "relevance": "Open-weight RTL code generation model using LLM-synthesized training data, demonstrating data-centric approaches to Verilog fine-tuning."
     },
     {
       "title": "AutoVCoder: A Systematic Framework for Automated Verilog Code Generation using LLMs",
       "authors": ["Mingzhe Gao", "Jieru Zhao"],
       "year": 2024,
       "doi": "10.1109/ICCD63220.2024.00033",
       "relevance": "Framework combining large-scale data curation (1M samples) with curriculum learning for Verilog generation."
     },
     {
       "title": "CodeV: Empowering LLMs with HDL Generation through Multi-Level Summarization",
       "authors": ["Yang Zhao", "Di Huang"],
       "year": 2025,
       "doi": "10.1109/TCAD.2025.3604320",
       "relevance": "Multi-level summarization approach for HDL code generation with MinHash deduplication for contamination control."
     },
     {
       "title": "MAGE: A Multi-Agent Engine for Automated RTL Code Generation",
       "authors": ["Yujie Zhao", "Hejia Zhang"],
       "year": 2025,
       "doi": "10.1109/DAC63849.2025.11133191",
       "relevance": "Multi-agent framework with four specialized agents for Verilog generation, representing the trend toward agentic approaches in hardware design."
     },
     {
       "title": "Large Language Models for Software Engineering: A Systematic Literature Review",
       "authors": ["Xinyi Hou", "Yanjie Zhao"],
       "year": 2024,
       "doi": "10.1145/3695988",
       "relevance": "Comprehensive SLR on LLMs for general software engineering that serves as a methodological baseline for this Verilog-specific survey."
     },
     {
       "title": "DeepRTL: Bridging Verilog Understanding and Generation with a Unified Representation Model",
       "authors": ["Yi Liu", "Changran Xu"],
       "year": 2025,
       "relevance": "Unified model for both Verilog understanding and generation with Chain-of-Thought annotation and multi-stage expert review achieving 90% annotation accuracy."
     },
     {
       "title": "VeriReason: Reinforcement Learning with Testbench Feedback for Reasoning-Enhanced Verilog Generation",
       "authors": ["Yiting Wang", "Guoheng Sun"],
       "year": 2025,
       "arxiv_id": "2505.11849",
       "relevance": "Combines reinforcement learning (GRPO) with AST-based rewards for reasoning-enhanced Verilog code generation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Provides a useful reference map of 34 instruction-tuned LLMs and 27 benchmarks for practitioners in hardware design automation, but is not directly usable as a tool or technique."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Findings confirm expected trends (LLM adoption growing, GPT dominant in closed-source, open-source catching up) without challenging any conventional wisdom."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Section 7.1 discusses security risks including hardware trojans from data poisoning and benchmark contamination, but these are reviewed rather than newly demonstrated."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; the paper is a neutral categorization of existing work."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or interactive tool is released alongside the survey."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from Zhejiang University and Singapore Management University are known in SE research but not household names; David Lo is well-known in the SE community."
     }
   }
 }