ai-research-survey

scan.json (27289B)

---

 {
   "paper": {
     "title": "ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design",
     "authors": [
       "Zhongkai Yu",
       "Chenyang Zhou",
       "Yichen Lin",
       "Hejia Zhang",
       "Haotian Ye",
       "Junxia Cui",
       "Zaifeng Pan",
       "Jishen Zhao",
       "Yufei Ding"
     ],
     "year": 2026,
     "venue": "Preprint (arXiv)",
     "arxiv_id": "2601.21448"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our code is available at https://github.com/zhongkaiyu/ChipBench.git' — a direct GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (44 Verilog generation cases, 89 debugging cases, 132 reference model cases) is released through the same GitHub repository. The paper describes gold Verilog implementations, test files, and prompts as part of the released benchmark."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is described in the paper. The paper mentions iVerilog and Verilator as tools but does not specify versions or dependency details for reproducing the evaluation environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the framework workflow is described conceptually (Section 2.4, Figure 3), the paper does not provide step-by-step reproduction instructions, README commands, or scripts to replicate the main experiments. The workflow description is at a design level, not an actionable instruction level."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2, 3, and 4 report only point estimates for pass@1, pass@5, and pass@10. No confidence intervals, error bars, or uncertainty measures are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'debugging capability exceeds generation capability', 'Python generation outperforms Verilog on simple modules') but provides no statistical significance tests — comparisons are based solely on comparing raw percentages."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context. For example, 'GPT-4o achieves ~65% pass@1 on RTLLM V2 and VerilogEval V2, but only 10.37% on our dataset' and 'MAGE achieves only 37.41% on ChipBench' vs. '95% accuracy on VerilogEval.' These provide enough context to understand the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why 44 Verilog generation cases, 89 debugging cases, or 132 reference model cases were chosen. No power analysis or discussion of whether these sample sizes are sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports pass@1, pass@5, and pass@10 but does not report standard deviation, variance, or spread across multiple runs. The evaluation appears to use sampling (temperature=0.85, top_p=0.95) but does not report run-to-run variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 13 models (GPT-3.5 through GPT-5.2, Claude family, Gemini family, DeepSeek family, Llama family) and the MAGE multi-agent system. It also compares ChipBench characteristics against existing benchmarks VerilogEval and RTLLM in Table 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include very recent models: GPT-5.2 (Dec 2025), Gemini 3 Flash (Dec 2025), Claude 4.5 Opus (Nov 2025), and MAGE. These represent state-of-the-art at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has multiple design decisions (three task categories, four bug types, three reference model languages, zero-shot vs one-shot debugging) but no ablation study examines how these design choices affect the benchmark's ability to discriminate between models. The zero-shot vs one-shot comparison (Section 3.3) is a task variant analysis, not a benchmark design ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports pass@1, pass@5, and pass@10, and also provides per-category breakdowns (CPU IP, Non-Self-Contained, Self-Contained) and per-bug-type breakdowns (arithmetic, assignment, state machine, timing). Cost analysis is also provided in Table 5."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper evaluating LLM code generation via automated test suites (iVerilog simulation, Verilator comparison). Human evaluation of LLM outputs is not relevant — the pass/fail evaluation is deterministic via simulation."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This paper introduces a new benchmark rather than training a model. There is no dev/test split needed — all 264+ test cases are the benchmark itself."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2, 3, and 4 all provide per-category breakdowns: Verilog generation is broken down by CPU IP, Non-Self-Contained, and Self-Contained; debugging is broken down by bug type (arithmetic, assignment, state machine, timing)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 discusses failure patterns in one-shot debugging ('most current LLMs lack the capability to interpret [waveform data] effectively'). Section 3.2 notes CPU IP design as the most challenging task with no model exceeding 22.22%. The paper also notes that 'all models failed when tasked with generating complete hierarchical designs' (Section 2.2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that one-shot debugging 'underperforms on 13' out of 21 model configurations compared to zero-shot (Section 3.3), which is a surprising negative finding. It also reports 0% pass rates for several models on CPU IP tasks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims Claude-4.5-opus achieves 30.74% on Verilog generation (confirmed in Table 2) and 13.33% on Python reference model generation (confirmed in Table 3, noting the abstract says 13.33% but the paper body says 15.93% for Claude Sonnet — the 13.33% is for Claude Opus, which is stated correctly). The abstract's claim of '95% pass rates' on existing benchmarks is supported by the MAGE reference in the introduction."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'Python generation outperforms Verilog on simple modules... likely due to Python's prevalence in LLM pre-training data' (Section 3.2) without evidence for this causal mechanism. It also claims 'most current LLMs lack the capability to interpret [waveform data] effectively' (Section 3.3) based on aggregate pass rate comparisons without controlling for confounds."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper title and framing ('AI-Aided Chip Design') suggests broad applicability, but the benchmark covers only 44 modules from specific open-source CPU IPs and competitive platforms. The paper does not bound its claims to these specific sources. Claims about 'industrial deployment requirements' are made without testing on actual industrial designs."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its main findings. For example, the lower performance on ChipBench vs VerilogEval could be partly due to longer prompts exceeding context windows, test harness differences, or other factors beyond module complexity. The one-shot debugging underperformance could be due to prompt format rather than LLM capability. No threats-to-validity or alternative explanations section is present."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 in the Appendix provides exact API model names with snapshot dates for all evaluated models (e.g., 'gpt-5.2-2025-12-11', 'claude-opus-4-5-20251101', 'gpt-4o-2024-11-20'). This is excellent version specification."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the prompting approach conceptually (Section 2.2: 'we manually wrote description prompts'; Section 2.3: debugging modes provide 'module description and buggy implementation') but the actual prompt text used for evaluation is not shown in the paper. The prompts are presumably in the GitHub repository but not presented in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states 'For all models, we use the hyperparameters recommended by VerilogEval: temperature=0.85 and top_p=0.95.' These are the key sampling parameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "The MAGE multi-agent system is tested but described only as 'a state-of-the-art multi-agent framework for Verilog generation' that 'employs dedicated agents for sampling, debugging, and decision-making.' The actual scaffolding details (tool descriptions, retry logic, decision criteria) are not described — the reader is referred to the original MAGE paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes how test cases were sourced ('open-source Online Judge platforms and established CPU IP projects'), categorized into three groups, and how prompts were manually written. Section 2.3 describes how debugging cases were constructed by manually injecting faults. The data pipeline from source to benchmark is well-documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. The paper has an 'Impact Statement' but it only says 'There are many potential societal consequences of our work, none which we feel must be specifically highlighted here.' The 'Take-Aways' section (Section 4) discusses future directions, not limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential issues with benchmark size (44 Verilog generation cases), selection bias in module choice, or limitations of the evaluation methodology."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its claims to the specific module types tested or acknowledge that 44 modules may not represent the full diversity of industrial chip designs."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (golden Verilog, test files, prompts, buggy implementations) is released at the GitHub repository. The raw evaluation data (model outputs) is not explicitly mentioned but the benchmark inputs are available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes sourcing from 'open-source Online Judge (OJ) platforms and established CPU IP projects.' Section 2.3 describes how debugging cases were created by manually injecting faults into golden modules. Table 7 lists all 44 modules with descriptions."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data comes from open-source code repositories and benchmarking of LLM APIs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from golden Verilog modules through bug injection (Section 2.3), test harness creation (Section 2.4), and reference model generation (Section 2.5) is documented with workflow diagrams (Figure 3). The benchmark construction pipeline is clear."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: UC San Diego Department of Computer Science and Engineering (authors 1, 3-8) and Columbia University (author 2)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The paper evaluates products from multiple commercial companies (Anthropic, OpenAI, Google, Meta) without disclosing any potential funding relationships."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the evaluated models. Table 6 lists release dates but not training data cutoffs. Since the benchmark uses modules from open-source projects, contamination is a relevant concern."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark modules are sourced from 'open-source Online Judge (OJ) platforms and established CPU IP projects' which may be in LLM training data. The paper does not discuss whether any of these modules could have been seen during model training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses open-source CPU IP projects and online judge problems that may have been available online before model training cutoffs. This contamination risk is not addressed. While the authors note the benchmark is new, the underlying source code it draws from is pre-existing open-source material."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides detailed cost analysis including input tokens, output tokens, cost in dollars, and cost per pass@1 for each model. Section 3.4 also reports costs for the training dataset generation tool (Figure 6)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports token consumption and API costs for each model. For the toolbox evaluation, Figure 6 shows total cost scaling with max debugging turns. While GPU hours are not reported (API-based evaluation), the API cost budget is clearly stated."
       }
     }
   },
   "claims": [
     {
       "claim": "State-of-the-art Claude-4.5-opus achieves only 30.74% pass@1 on Verilog generation and 13.33% on Python reference model generation on ChipBench.",
       "evidence": "Table 2 shows Claude 4.5 Opus at 30.74% average pass@1 for Verilog generation. Table 3 shows Claude 4.5 Opus at 13.33% average pass@1 for Python reference model generation.",
       "supported": "strong"
     },
     {
       "claim": "MAGE, which exceeds 95% accuracy on VerilogEval, achieves only 37.41% on ChipBench.",
       "evidence": "Table 2 shows MAGE (DeepSeek-V3) at 37.41% pass@1 average. The 95% on VerilogEval claim is attributed to Zhao et al., 2025b in the introduction.",
       "supported": "strong"
     },
     {
       "claim": "LLMs consistently perform better on Verilog debugging than generation when given the same module.",
       "evidence": "Section 3.2 compares Claude-opus at 47.45% debugging (Table 4) vs 30.74% generation (Table 2). This pattern holds across models, with 5%-20% higher pass rates stated for debugging.",
       "supported": "moderate"
     },
     {
       "claim": "One-shot debugging (with waveform data) does not consistently outperform zero-shot debugging, suggesting LLMs lack waveform interpretation capability.",
       "evidence": "Section 3.3 and Figure 5 report that one-shot outperforms zero-shot on 8 models but underperforms on 13.",
       "supported": "moderate"
     },
     {
       "claim": "For simple self-contained modules, LLMs achieve 5%-20% higher accuracy on Python reference model generation compared to Verilog generation.",
       "evidence": "Comparing Table 3 self-contained columns with Table 2 self-contained columns shows higher Python pass rates for most models on simple modules (e.g., Claude Opus: 40% Python vs 36.67% Verilog).",
       "supported": "moderate"
     },
     {
       "claim": "The automated toolbox can generate approximately n/2 Python reference models from n Verilog training samples with multi-turn debugging.",
       "evidence": "Section 3.4 reports 2,206 verified models from 10,000 samples (22%), and Figure 6 shows pass rate plateauing around 50% with higher max turns on a 100-sample subset.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "ChipBench introduces a benchmark with 264+ test cases spanning Verilog generation, debugging, and reference model generation that reveals significant performance gaps in current LLMs for chip design tasks. The best-performing model (Claude 4.5 Opus) achieves only 30.74% pass@1 on Verilog generation and 13.33% on Python reference model generation, demonstrating that existing saturated benchmarks with 95%+ pass rates overestimate LLM capabilities for industrial deployment. LLMs show stronger debugging ability than generation ability, and one-shot debugging with waveform data does not consistently improve performance, suggesting LLMs struggle with waveform interpretation.",
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "The benchmark draws modules from open-source CPU IPs and online judge platforms. These are public repositories that are very likely in the training data of the evaluated models. No contamination analysis, temporal split, or canary string approach is used. This is especially concerning for the self-contained modules from OJ platforms, which resemble common coding contest problems."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity section. For a benchmark paper, key missing discussions include: selection bias in module choice, small benchmark size (44 Verilog generation cases), representativeness of OJ and open-source CPU IP problems for industrial workflows, and the gap between the benchmark's claim to represent 'industrial complexity' and its actual source material."
     },
     {
       "flag": "Small and potentially unrepresentative benchmark",
       "detail": "With only 44 Verilog generation cases, 6 non-self-contained cases, and 9 CPU IP cases, per-category sample sizes are very small. The CPU IP category has only 9 cases, yet claims about model capabilities on industrial designs are drawn from these. Individual category pass rates could shift dramatically with a few additional cases."
     },
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "Despite using stochastic sampling (temperature=0.85, top_p=0.95), the paper reports no confidence intervals, standard deviations, or significance tests. The pass@5 and pass@10 metrics require multiple samples, but variance across sampling runs is not reported. Differences between models could be within noise."
     },
     {
       "flag": "Unbounded generalization claims",
       "detail": "The paper claims the benchmark reflects 'industrial deployment requirements' and 'real-world industrial scenarios,' but the modules come from open-source CPU IPs and competitive platforms. Industrial Verilog modules typically exceed 10,000 lines (as the paper itself notes), while ChipBench averages 61.7 lines. The gap between the benchmark and true industrial complexity is acknowledged but not reflected in the paper's claims."
     }
   ],
   "cited_papers": [
     {
       "title": "VerilogEval: Evaluating Large Language Models for Verilog Code Generation",
       "authors": ["Mingjie Liu", "Nathaniel Pinckney", "Brucek Khailany", "Haoxing Ren"],
       "year": 2023,
       "relevance": "Seminal benchmark for LLM Verilog generation evaluation; ChipBench directly compares against and extends this benchmark."
     },
     {
       "title": "MAGE: A Multi-Agent Engine for Automated RTL Code Generation",
       "authors": ["Yiliang Zhao", "Hejia Zhang", "Hanyi Huang", "Zhongkai Yu", "Jishen Zhao"],
       "year": 2025,
       "relevance": "State-of-the-art multi-agent framework for Verilog generation evaluated as a baseline, achieving 95% on VerilogEval but only 37% on ChipBench."
     },
     {
       "title": "RTLLM: An Open-Source Benchmark for Design RTL Generation with Large Language Model",
       "authors": ["Yao Lu", "Shang Liu", "Qijun Zhang", "Zhiyao Xie"],
       "year": 2024,
       "relevance": "Competing benchmark for LLM-based RTL generation that ChipBench claims to supersede due to saturation."
     },
     {
       "title": "Revisiting VerilogEval: A Year of Improvements in Large-Language Models for Hardware Code Generation",
       "authors": ["Nathaniel Pinckney", "Christopher Batten", "Mingjie Liu", "Haoxing Ren", "Brucek Khailany"],
       "year": 2025,
       "relevance": "Updated version of VerilogEval benchmark; ChipBench inherits its evaluation framework and compares against its design choices."
     },
     {
       "title": "RealBench: Benchmarking Verilog Generation Models with Real-World IP Designs",
       "authors": ["Peng Jin", "Dong Huang", "Cheng Li"],
       "year": 2025,
       "relevance": "Contemporary benchmark using real-world IP designs; described as complementary to ChipBench in evaluating LLM chip design capabilities."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "One of the evaluated models with reasoning capabilities; relevant to understanding LLM reasoning in code generation tasks."
     },
     {
       "title": "A Survey on Code Generation with LLM-based Agents",
       "authors": ["Yihong Dong", "Xue Jiang", "Jian Qian"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Survey of LLM-based agent code generation techniques, providing context for agentic approaches to chip design."
     },
     {
       "title": "VeriGen: A Large Language Model for Verilog Code Generation",
       "authors": ["Shailja Thakur", "Baleegh Ahmad", "Hammond Pearce", "Benjamin Tan", "Brendan Dolan-Gavitt", "Ramesh Karri", "Siddharth Garg"],
       "year": 2024,
       "relevance": "Prior work on LLM-based Verilog generation providing training datasets (50k samples) used with ChipBench toolbox."
     },
     {
       "title": "QiMeng-CodeV-R1: Reasoning-Enhanced Verilog Generation",
       "authors": ["Yutong Zhu", "Dong Huang", "Haoran Lyu"],
       "year": 2025,
       "relevance": "Provides training dataset (87k samples) used for toolbox evaluation and contains the CodeV-R1 dataset used for reference model generation experiments."
     },
     {
       "title": "Pro-V: An Efficient Program Generation Multi-Agent System for Automatic RTL Verification",
       "authors": ["Yiliang Zhao", "Zhifang Wu", "Hejia Zhang", "Zhongkai Yu"],
       "year": 2025,
       "arxiv_id": "2506.12200",
       "relevance": "Multi-agent system for RTL verification including reference model generation as a sub-task; ChipBench argues it benefits from accurate reference models."
     },
     {
       "title": "Large Language Models for EDA: Future or Mirage?",
       "authors": ["Zhiyao He", "Yu Pu", "Hao Wu", "Tiancheng Qiu", "Bei Yu"],
       "year": 2025,
       "relevance": "Comprehensive survey on LLMs for electronic design automation, providing broader context for the chip design benchmarking landscape."
     }
   ]
 }