ai-research-survey

scan-v5.json (22311B)

---

 {
   "scan_version": 5,
   "paper_type": "survey",
   "paper": {
     "title": "Large Language Model for Verilog Code Generation: Literature Review and the Road Ahead",
     "authors": [
       "Guang Yang",
       "Wei Zheng",
       "Dong Liang",
       "Peng Hu",
       "Yukui Yang",
       "Shaohang Peng",
       "Zhenghan Li",
       "Jiahui Feng",
       "Xiao Wei",
       "Kexin Sun",
       "Deyuan Ma",
       "Haotian Cheng",
       "Yiheng Shen",
       "Xiang Chen",
       "Xing HU",
       "Terry Yue Zhuo",
       "David Lo"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2512.00020",
     "doi": "10.48550/arXiv.2512.00020"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to analyze 102 papers across 4 RQs covering LLMs used, datasets/metrics, optimization techniques, and alignment approaches; all four RQs have dedicated sections with supporting evidence from the corpus.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is a descriptive systematic review making no causal claims; trend observations (e.g., open-source adoption growing faster) are framed as patterns, not causal mechanisms.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The survey explicitly scopes to 2020-2025 and Verilog code generation specifically; Table 1 delineates what prior surveys cover vs. this paper's contribution, and claims stay within the 102-paper corpus.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Trends such as GPT dominance or the rapid growth of open-source LLM adoption are reported without considering alternative explanations such as publication venue bias or citation inflation effects.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 explicitly distinguishes similarity-based metrics (proxies), execution-based metrics (direct functional correctness), and LLM-as-judge metrics, noting that functional-pass@k is the most direct measure.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 'Threats to Validity' is a dedicated section addressing three specific threats: paper search omission, study selection bias, and categorization/analysis bias.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific with mitigations: 'fast-paced LLM research may lead to overlooking emerging work, mitigated through rigorous keyword selection and multi-stage filtering' and 'reduction involved subjective quality assessments, mitigated by strict inclusion criteria and a Likert-scale framework requiring 12/15 threshold.'",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Temporal scope (2020-2025), language focus (Verilog only), and task focus (code generation, not verification or testing without generation) are explicitly stated and distinguished from broader EDA surveys in Table 1.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 11 (Acknowledgments) discloses partial funding from the National Natural Science Foundation of China (NSFC, No. 62141208).",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All 17 author affiliations are listed in the paper header: Zhejiang University, Northwestern Polytechnical University, Nantong University, Monash University, and Singapore Management University.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSFC is a Chinese government science foundation with no financial stake in any of the commercial tools, proprietary models, or industry datasets reviewed in this survey.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided; the paper only discloses the NSFC grant without any formal declaration of personal financial interests, patents, or consulting relationships.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 formally defines Verilog, HDL, EDA workflow phases, RTL, and specifies LLM-based Verilog generation as a formal mapping function f_θ: (D,I) → V; Section 4 distinguishes Base LLMs from IT LLMs with explicit criteria.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contribution bullets in the introduction state: (1) first systematic literature review on LLM-based Verilog generation analyzing 102 papers, (2) comprehensive taxonomy across 4 RQs, and (3) key limitations and roadmap.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 explicitly compares with six prior surveys (Jiang et al. 2024, Joel et al. 2024, Chen et al. 2025, Fang et al. 2025, He et al. 2024, Chen et al. 2024) showing specific gaps each fails to address that this survey fills.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 fully describes the QGS strategy with six named databases, two explicit keyword sets (25 total terms), temporal filters (2020-2025), and a three-stage filtering process with snowballing; the strategy could be independently replicated.",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 explicitly lists 3 inclusion criteria and 6 exclusion criteria (e.g., 'short papers < 5 pages,' 'paper is a literature review or survey,' 'LLMs mentioned only in future work'); paper counts at each stage are reported.",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly follows Kitchenham et al.'s SEGRESS SLR guidelines [59] and the Quasi-Gold Standard (QGS) strategy [133], which are established structured review protocols with planning, conducting, and analysis phases.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2.1 provides all search terms verbatim: 12 keywords for Verilog code generation (e.g., 'Verilog,' 'HDL,' 'RTL,' 'EDA,' 'FPGA') and 13 keywords for LLMs (e.g., 'LLM,' 'GPT,' 'fine-tuning,' 'prompt engineering').",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2.2 explicitly names all six databases searched: IEEE Xplore, ACM Digital Library, ScienceDirect, Web of Science, SpringerLink, and arXiv.",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.3.1 documents paper counts at each stage: Stage 1 (short/duplicate removal) → 5,172; Stage 2 (content filtering) → 687; Stage 3 (full-text) → 124; QAC assessment → 85; snowballing → +15 = 102 final.",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": true,
           "justification": "The 2020 start date is justified by citing the field's inception paper (DAVE, 2020); the Verilog-specific focus is justified by the gap analysis in Table 1 showing no prior survey covers this niche comprehensively.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": false,
           "justification": "The survey presents a unified narrative of progress across all four RQs; it does not explicitly identify cases where reviewed papers report contradictory results or where different approaches yield conflicting performance conclusions.",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 presents a five-criterion Quality Assessment Criteria (QAC) framework evaluated on a 0-3 Likert scale with a minimum threshold of 12/15 points; preprints are assessed separately on non-venue criteria.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss publication bias; it does not acknowledge that published LLM papers in this domain systematically favor reporting positive benchmark results, which likely skews the field's apparent progress.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": false,
           "justification": "Synthesis is purely descriptive with counts and percentages (e.g., 204 open-source usages = 53.3%); there is no formal meta-analysis, effect size aggregation, statistical comparison across approaches, or vote counting with confidence intervals.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": true,
           "justification": "The three-stage roadmap in Section 8.2 derives each recommendation from specific limitations identified in RQ1-RQ4 (e.g., 'small-scale benchmarks <100 samples' → 'construct benchmarks exceeding 1,000 samples with PPA truth').",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "LLM-based Verilog generation research grew from 1 paper in 2020 to 66 papers by September 2025, representing exponential growth.",
       "evidence": "Figure 4b shows exact annual counts: 1 (2020), 0 (2021), 0 (2022), 6 (2023), 29 (2024), 66 (2025).",
       "supported": "strong"
     },
     {
       "claim": "GPT-series models dominate closed-source LLM adoption with 149 of 179 total mentions (83.2%) across surveyed papers.",
       "evidence": "Table 4 provides exact usage counts per LLM family by year; GPT: 149 total vs. Claude: 19, Gemini: 5, Others: 6.",
       "supported": "strong"
     },
     {
       "claim": "Open-source LLMs slightly outnumber closed-source in total usage across the surveyed literature (204 vs. 179 instances, 53.3% vs. 46.7%).",
       "evidence": "Table 4 aggregates total usages by open vs. closed source across all 102 papers, with Llama (64) and DeepSeek (54) leading open-source.",
       "supported": "strong"
     },
     {
       "claim": "55.9% of domain-specific instruction-tuned LLMs for Verilog release public model weights.",
       "evidence": "Table 5 lists 34 IT LLMs, 19 open-weight and 15 closed-weight; 19/34 = 55.9%, with open-weight models linked to HuggingFace/GitHub repositories.",
       "supported": "strong"
     },
     {
       "claim": "Most instruct-tuning datasets lack executable testbench supervision, creating a gap between training and evaluation pipelines.",
       "evidence": "Tables 6 and 7 show nearly all benchmarks include testbenches (16/18 open), while only a few instruct-tuning datasets (VeriCoder-Origen, VeriReason-Data) include testbench supervision.",
       "supported": "strong"
     },
     {
       "claim": "Existing benchmarks are small-scale (<100 samples) and neglect system-level complexity such as pipelined datapaths and SoCs.",
       "evidence": "Section 8.1 makes this claim, but Table 6 shows CVDP (783 samples), VeriThoughts (291), and RTLRepo_test (1.17K), making the '<100 samples' generalization partially inaccurate.",
       "supported": "weak"
     },
     {
       "claim": "Functional-pass@k has become the dominant metric for evaluating Verilog code generation quality.",
       "evidence": "Section 5.2 RQ2 summary explicitly states 'functional-pass@k becoming the dominant indicator of correctness'; trend analysis confirms shift from similarity-based to execution-based metrics.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "meta-analysis",
     "qualitative"
   ],
   "key_findings": "This systematic review of 102 papers (2020-2025) finds the LLM-based Verilog code generation field has grown explosively, from one paper in 2020 to 66 by September 2025, with GPT-series dominating closed-source adoption (83.2%) and Llama/DeepSeek leading open-source. Functional-pass@k has emerged as the dominant evaluation metric, displacing text similarity measures, but a critical training-evaluation gap persists because nearly all instruct-tuning datasets lack executable testbench supervision while evaluation benchmarks uniformly require it. The field faces four alignment challenges (security, efficiency/PPA, copyright, hallucinations) that remain largely unsolved, and no current system integrates into industrial EDA workflows, leaving a significant gap between experimental prototypes and production-ready hardware design assistance.",
   "red_flags": [
     {
       "flag": "No publication bias acknowledgment",
       "detail": "The survey does not discuss the systematic tendency of LLM papers to report positive results, which likely inflates apparent progress across all reviewed techniques and benchmarks."
     },
     {
       "flag": "No conflicting findings identified",
       "detail": "The synthesis presents a unified narrative without identifying where reviewed papers contradict each other on methodology effectiveness, benchmark validity, or performance rankings across approaches."
     },
     {
       "flag": "No quantitative synthesis",
       "detail": "Despite reviewing 102 papers with reported performance numbers on shared benchmarks (VerilogEval, RTLLM), the synthesis is purely descriptive; no effect sizes, pooled estimates, or statistical comparisons across approaches are provided."
     },
     {
       "flag": "Benchmark characterization overstated",
       "detail": "Section 8.1 claims benchmarks are 'small-scale (<100 samples)' but Table 6 shows CVDP (783), VeriThoughts (291), and RTLRepo_test (1.17K) — the characterization is inaccurate for roughly a third of listed benchmarks."
     },
     {
       "flag": "Financial interests undeclared",
       "detail": "No competing interests statement is provided despite authors being affiliated with institutions that may have collaborations with commercial EDA vendors or LLM API providers mentioned in the survey."
     }
   ],
   "cited_papers": [
     {
       "title": "SEGRESS: Software Engineering Guidelines for REporting Secondary Studies (Kitchenham et al. 2023)",
       "relevance": "Methodological foundation for the systematic literature review protocol used in this paper"
     },
     {
       "title": "VerilogEval: Evaluating Large Language Models for Verilog Code Generation (Liu et al. 2023)",
       "relevance": "The primary benchmark for Verilog code generation evaluation, most widely used across the reviewed literature"
     },
     {
       "title": "VeriGen: A Large Language Model for Verilog Code Generation (Thakur et al. 2024)",
       "relevance": "Foundational early paper establishing benchmarks and fine-tuning baselines for the field"
     },
     {
       "title": "RTLCoder: Fully Open-Source and Efficient LLM-Assisted RTL Code Generation Technique (Liu et al. 2025)",
       "relevance": "Major open-source fine-tuning approach representing the state of the art in data-centric supervised training"
     },
     {
       "title": "CraftRTL: High-quality Synthetic Data Generation for Verilog Code Models (Liu et al. 2025, ICLR)",
       "relevance": "ICLR 2025 paper on correct-by-construction synthetic data, cited as key advance in training data quality"
     },
     {
       "title": "DAVE: Deriving Automatically Verilog from English (Pearce et al. 2020)",
       "relevance": "The first paper in the field, marking the inception of LLM-based Verilog generation research used to justify the 2020 scope boundary"
     },
     {
       "title": "Large Language Models for Software Engineering: A Systematic Literature Review (Hou et al. 2024)",
       "relevance": "Parallel SLR on general software engineering that this survey explicitly positions against in Table 1"
     },
     {
       "title": "VerilogCoder: Autonomous Verilog Coding Agents with Graph-based Planning and AST-based Waveform Tracing (Ho et al. 2025, AAAI)",
       "relevance": "AAAI 2025 multi-agent system representing the frontier of agentic Verilog generation with EDA-tool feedback"
     },
     {
       "title": "Identifying relevant studies in software engineering — QGS methodology (Zhang et al. 2011)",
       "relevance": "Provides the Quasi-Gold Standard search strategy used as the core methodology for study identification"
     },
     {
       "title": "DeepRTL: Bridging Verilog Understanding and Generation with a Unified Representation Model (Liu et al. 2025, ICLR)",
       "relevance": "ICLR 2025 knowledge-enhanced tuning approach combining understanding and generation tasks"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Hardware engineers and AI researchers can directly use the taxonomies of 27 benchmarks, 34 training datasets, and categorized techniques to navigate and advance LLM-based Verilog design."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Mostly confirms expected trends (GPT dominance, open-source growth); the training-evaluation gap around testbench supervision is notable but not widely counterintuitive to the field."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Section 7.1 discusses hardware trojans and security vulnerabilities in LLM-generated Verilog, but this is a secondary concern rather than the paper's central thesis."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Standard academic survey with no controversy, failed replications, or competing factions highlighted; the synthesis is consistently collaborative in tone."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Pure literature survey with no tool, dataset, or interactive demo introduced; practitioners cannot immediately try anything from this paper alone."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "David Lo (Singapore Management University) is a well-known software engineering researcher; the venue is ACM Computing Surveys (submitted), a prestigious venue, but no major industry lab is a co-author."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "33833631",
         "title": "Multi-scale gigapixel microscopy using a multi-camera array microscope",
         "points": 12,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=33833631"
       },
       {
         "hn_id": "42277470",
         "title": "Stable, Fast, Automatic Learning Algorithm for Predictive Coding Networks [pdf]",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42277470"
       },
       {
         "hn_id": "45979786",
         "title": "Semi-Supervised Preference Optimization with Limited Feedback",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45979786"
       },
       {
         "hn_id": "45966047",
         "title": "VRScout: Towards Real-Time, Autonomous Testing of Virtual Reality Games",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45966047"
       },
       {
         "hn_id": "45917707",
         "title": "Probing Knowledge Holes in Unlearned LLMs",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45917707"
       },
       {
         "hn_id": "4448824",
         "title": "Letting daylight in: review reviewers and maximize transparency in science",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=4448824"
       },
       {
         "hn_id": "46004769",
         "title": "Repetitive vs. Non-repetitive Lidar scanning pattern for roadside perception",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46004769"
       },
       {
         "hn_id": "45817907",
         "title": "Fix: Externalizing network I/O in serverless computing [pdf]",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45817907"
       },
       {
         "hn_id": "4095443",
         "title": "Modeling Barn Owls to make ultra precise localization systems",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=4095443"
       },
       {
         "hn_id": "33984298",
         "title": "Paper – Educational Opportunities and Challenges of AI Code Generation",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=33984298"
       }
     ],
     "top_points": 12,
     "total_points": 26,
     "total_comments": 1
   }
 }