ai-research-survey

scan.json (20389B)

---

 {
   "paper": {
     "title": "Revolution or Hype? Seeking the Limits of Large Models in Hardware Design",
     "authors": ["Qiang Xu", "Leon Stok", "Rolf Drechsler", "Xi Wang", "Grace Li Zhang", "Igor L. Markov"],
     "year": 2025,
     "venue": "2025 IEEE/ACM International Conference On Computer Aided Design (ICCAD)",
     "arxiv_id": "2509.04905",
     "doi": "10.1109/ICCAD66269.2025.11240750"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis", "qualitative"],
   "key_findings": "This ICCAD 2025 panel paper surveys the state of Large Language Models (LLMs) and Large Circuit Models (LCMs) in electronic design automation. It identifies key opportunities (RTL generation, verification acceleration, PPA optimization) and challenges (hallucination, data scarcity, explainability). The panelists converge on a division of labor: LLMs for interpreting high-level intent ('the What') and LCMs for circuit-native optimization ('the How'), with formal verification as the indispensable trust anchor.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code or repository is released. This is a survey/panel paper but could have released analysis scripts or curated reference lists."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset or curated corpus of surveyed papers is released."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments are performed; this is a qualitative survey/panel paper."
       },
       "reproduction_instructions": {
         "applies": false,
         "answer": false,
         "justification": "No experiments to reproduce; this is a position/panel paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative experiments are conducted. The paper is a qualitative survey and panel synthesis."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No statistical comparisons are made; this is a qualitative overview paper."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with quantitative results are presented."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No empirical sample is collected or analyzed."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs are performed."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not compare against prior surveys of LLMs/LCMs for EDA. It references prior work but does not position itself against comparable survey papers."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No baselines are included, so contemporaneity is inapplicable."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system or method with components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative evaluation is performed."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate; this is a qualitative panel synthesis."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No datasets or test sets are used."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides structured breakdowns by task type: Section II-A covers LLMs for hardware design (code generation, verification, optimization) and Section II-B covers LCMs. Fig. 2 and Fig. 3 provide detailed chronological breakdowns by application area."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV discusses challenges extensively: hallucination (IV-A), semantic gap in circuit representation (IV-B), data scarcity (IV-C), and explainability (IV-D). Section V-B further discusses failure modes and reliability concerns from each panelist."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses limitations and negative aspects: Markov notes LLMs struggle with arithmetic (ref [43]), reasoning quality declines with task size (ref [44]), and productivity gains may be overstated (ref [45]). The entire framing is 'Revolution or Hype?' acknowledging potential failure of the paradigm."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the paper 'critically examines the practical capabilities, fundamental limitations, and future prospects' and provides 'an authoritative overview.' The body delivers on this through structured sections on background (II), opportunities (III), challenges (IV), expert opinions (V), and recommendations (VI)."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It surveys and synthesizes opinions about the state of a field without asserting causal relationships."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly bounds its scope to hardware design / EDA workflows. It distinguishes between LLM and LCM applicability for different stages of the design flow (Fig. 4) and notes limitations are specific to the EDA domain (data scarcity, IP concerns)."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative survey/panel paper with no empirical results requiring alternative explanations."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "No measurements are taken; this is a qualitative position paper."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models are used in experiments; this is a survey paper."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used; this is a survey paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the surveyed papers were selected. There is no description of a search strategy, inclusion/exclusion criteria, or filtering pipeline. Papers appear to be selected ad-hoc based on the panelists' expertise."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Section IV discusses challenges of the technology being surveyed, not limitations of the paper itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to the validity of this survey are discussed. The paper does not acknowledge that panelist selection, scope of surveyed work, or the qualitative nature of expert opinions may bias its conclusions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states it focuses on LLMs and LCMs for hardware/EDA design, synthesizing perspectives from the ICCAD 2025 panel (Introduction). The scope is explicitly bounded to hardware design, not software or general AI."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (e.g., list of all surveyed papers, selection criteria, or panelist transcripts) is made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the literature was collected or how panelist contributions were solicited and integrated."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants beyond the named panelists, who are introduced in biographies. This is not a study with recruited participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of how the survey was assembled—no search queries, databases searched, or filtering stages described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: CUHK, IBM, University of Bremen/DFKI, Southeast University, TU Darmstadt, and Synopsys. Industry affiliations (IBM, Synopsys) are transparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. Two authors are from major EDA industry players (IBM, Synopsys) who have commercial interest in AI-for-EDA outcomes. This potential conflict is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. Markov is from Synopsys and references Synopsys products (DSO.ai, ChipArchitect, EUCLIDE, KNOWLEDGE ASSISTANT, RUN ASSISTANT) multiple times. Stok is from IBM. No disclosure of financial interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey/panel paper with no computational experiments."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey/panel paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "No structured review protocol is followed. Papers are selected ad-hoc by panelists. No search strategy, database queries, or PRISMA flow diagram is provided."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The surveyed papers are not assessed for methodological quality. All referenced work is treated equally regardless of rigor. For example, arxiv preprints and peer-reviewed conference papers are cited without distinction."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Publication bias is not discussed. Markov briefly mentions data leakage in benchmarks (Section VI) but publication bias in the surveyed literature itself is not addressed."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs and LCMs serve complementary roles: LLMs for interpreting high-level intent ('the What') and LCMs for circuit-native optimization ('the How').",
       "evidence": "Section V-A: Qiang Xu proposes a division of labor between LLMs (natural language interface for design intent) and LCMs (expert engineer for PPA optimization). Leon Stok reinforces this with practical use cases.",
       "supported": "moderate"
     },
     {
       "claim": "LLM hallucination is a critical barrier to industrial EDA adoption, as even 0.1% error rates can cause catastrophic silicon failures.",
       "evidence": "Section IV-A and V-B: Multiple panelists (Zhang, Drechsler, Stok) identify hallucination as a key blocker. Drechsler notes 0.1% error tolerance in safety-critical design.",
       "supported": "moderate"
     },
     {
       "claim": "Standard transformer models are generally inefficient at representing and reasoning over high-precision numerical values needed for EDA.",
       "evidence": "Section V-A: Markov cites ref [43] (McLeish et al.) showing transformers struggle with arithmetic, and ref [44] showing sharp declines in output quality as task size increases.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs are already in commercial use for targeted EDA tasks such as converting natural-language specs into structured formats and generating RTL code snippets.",
       "evidence": "Section V-B: Markov describes Synopsys KNOWLEDGE ASSISTANT, RUN ASSISTANT, and EUCLIDE IDE as commercially deployed LLM-based tools.",
       "supported": "moderate"
     },
     {
       "claim": "Perceived productivity gains from LLM code assistants may not match actual productivity, with engineers often rewriting or discarding generated code.",
       "evidence": "Section V-A: Markov cites ref [45] (METR study from McKinsey) showing mismatch between perceived and actual productivity gains.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Undisclosed conflicts of interest",
       "detail": "Igor Markov (Synopsys) references multiple Synopsys products (DSO.ai, ChipArchitect, KNOWLEDGE ASSISTANT, RUN ASSISTANT, EUCLIDE) without explicit conflict-of-interest disclosure. Leon Stok is VP of EDA at IBM. Both have commercial stakes in AI-for-EDA outcomes."
     },
     {
       "flag": "No structured review methodology",
       "detail": "The paper surveys dozens of papers but follows no systematic review protocol. Paper selection appears driven by panelist familiarity and self-citation rather than a reproducible search strategy."
     },
     {
       "flag": "No quality assessment of surveyed work",
       "detail": "All referenced papers are treated with equal weight. ArXiv preprints and peer-reviewed publications are cited interchangeably without assessing their methodological rigor."
     },
     {
       "flag": "Self-promotion through survey",
       "detail": "Panelists frequently cite their own work: Xu cites DeepGate/DeepGate3/DeepCell/DeepRTL (his group), Wang cites ChatCPU/MEIC/UVLLM/VeriDebug (his group), Drechsler cites AutoBench/CorrectBench (his group). The survey structure gives each panelist a platform to advocate for their research agenda."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring the impact of early-2025 AI on experienced open-source developer productivity",
       "authors": ["J. Becker", "N. Rush", "E. Barnes", "D. Rein"],
       "year": 2025,
       "relevance": "RCT measuring actual vs. perceived AI coding assistant productivity gains, directly relevant to survey's methodology quality assessment."
     },
     {
       "title": "ChipNeMo: Domain-adapted LLMs for Chip Design",
       "authors": ["M. Liu"],
       "year": 2023,
       "arxiv_id": "2311.00176",
       "relevance": "Domain-adapted LLM for hardware design, key example of LLM specialization for EDA tasks."
     },
     {
       "title": "VerilogCoder: Autonomous Verilog coding agents with graph-based planning and abstract syntax tree (AST)-based waveform tracing tool",
       "authors": ["C.-T. Ho", "H. Ren", "B. Khailany"],
       "year": 2025,
       "relevance": "Agentic LLM pipeline for hardware code generation with tool-in-the-loop verification."
     },
     {
       "title": "The illusion of thinking: Understanding the strengths and limitations of reasoning models via the lens of problem complexity",
       "authors": ["P. Shojaee"],
       "year": 2025,
       "arxiv_id": "2506.06941",
       "relevance": "Demonstrates sharp declines in LLM reasoning quality as task complexity increases, relevant to AI capability evaluation."
     },
     {
       "title": "Transformers can do arithmetic with the right embeddings",
       "authors": ["S. McLeish"],
       "year": 2024,
       "relevance": "Analyzes fundamental limitations of transformers on numerical reasoning tasks, relevant to LLM capability boundaries."
     },
     {
       "title": "RTLCoder: Fully open-source and efficient LLM-assisted RTL code generation technique",
       "authors": ["S. Liu", "W. Fang", "Y. Lu", "J. Wang", "Q. Zhang", "H. Zhang", "Z. Xie"],
       "year": 2024,
       "relevance": "Open-source LLM for RTL code generation, key benchmark in LLM-for-hardware-design evaluation."
     },
     {
       "title": "AutoChip: Automating HDL generation using LLM feedback",
       "authors": ["S. Thakur", "J. Blocklove", "H. Pearce", "B. Tan", "S. Garg", "R. Karri"],
       "year": 2023,
       "arxiv_id": "2311.04887",
       "relevance": "LLM-based hardware code generation with iterative feedback loops, early agentic approach to EDA."
     },
     {
       "title": "Large circuit models: opportunities and challenges",
       "authors": ["L. Chen", "Y. Chen", "Z. Chu"],
       "year": 2024,
       "relevance": "Foundation paper defining the Large Circuit Model concept for AI-native EDA."
     },
     {
       "title": "Accurate predictions on small data with a tabular foundation model",
       "authors": ["N. Hollmann"],
       "year": 2025,
       "relevance": "Demonstrates synthetic data strategy for foundation model training, cited as alternative to scarce domain data."
     },
     {
       "title": "A survey of research in large language models for electronic design automation",
       "authors": ["J. Pan", "G. Zhou", "C.-C. Chang", "I. Jacobson", "J. Hu", "Y. Chen"],
       "year": 2025,
       "relevance": "Comprehensive survey of LLM applications in EDA, direct comparator to this panel paper."
     }
   ]
 }