ai-research-survey

scan.json (20181B)

---

 {
   "paper": {
     "title": "Comprehensive Verilog Design Problems: A Next-Generation Benchmark Dataset for Evaluating Large Language Models and Agents on RTL Design and Verification",
     "authors": ["Nathaniel Pinckney", "Chenhui Deng", "Chia-Tung Ho", "Yun-Da Tsai", "Mingjie Liu", "Wenfei Zhou", "Brucek Khailany", "Haoxing Ren"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2506.14074"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive link is provided in the paper. The paper describes infrastructure but does not release it."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset download link is provided. The benchmark is described but no URL for accessing the 783 problems is given."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies Docker containers for evaluation, names specific open-source tools (Icarus Verilog, Yosys, Verilator) with versions, and mentions Cadence Xcelium for commercial tasks."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The evaluation flow is described at a high level (Figure 1) but not with actionable commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 2-4 report only point estimates (e.g., '33.56%') with no confidence intervals or error bars despite using n=5 samples."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares models (e.g., Claude 3.7 Sonnet vs GPT 4.1) without any statistical significance tests. Differences are stated as raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes reported. Differences between models are stated as raw percentage points (e.g., '8% lower') without formal effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "n=5 samples per problem is stated but not justified. No power analysis or rationale for why 5 samples is sufficient for reliable pass@1 estimation."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures reported across the n=5 samples. Only point estimates of pass@1 are given."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple models are compared (Claude 3.7 Sonnet, GPT 4.1, GPT o1, o4-mini, Llama 3.1 405B/70B). Prior benchmark results (VerilogEval v2) are referenced for context."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include very recent releases: GPT 4.1 (April 2025), o4-mini (April 2025), Claude 3.7 Sonnet (2025). These represent state-of-the-art."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study on benchmark design choices (e.g., effect of quality filtering on results is mentioned but not systematically ablated as a design decision)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Pass@1 for code generation, BLEU for correspondence tasks, and LLM-based judging for Q&A tasks. Multiple metrics across different task types."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs. Code generation is evaluated by automated test harnesses, and comprehension is scored by BLEU and LLM-based judging (GPT o4-mini)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper; the entire dataset serves as a test set for evaluating models. There is no training/tuning involved."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2-4 provide per-category breakdowns across all 13 task categories (cid02-cid16). Table 5 provides detailed per-category failure analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 provides systematic failure analysis with clustering of failure types, visualizations (Figures 2-3), and specific failure examples in Appendix A."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that models struggle significantly on verification tasks (e.g., 0% pass@1 for Claude 3.7 Sonnet on agentic cid13), and that extended thinking mode provides minimal benefit."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims ('no more than 34% pass@1', 'agentic tasks are particularly difficult') are supported by Tables 2-3 showing Claude 3.7 Sonnet at 33.56% and lower agentic scores."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper suggests causal explanations (e.g., 'This discrepancy may stem from the more procedural and imperative nature of testbench code') without controlled experiments to verify."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section (Section 6) explicitly bounds scope: 'limited to standard hardware design and verification tasks', acknowledges Q&A tasks don't sufficiently challenge LLMs, and notes agentic context is oracle context."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for the performance gaps. For instance, whether low verification pass rates are due to benchmark difficulty, prompt quality, or model limitations is not explored."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'Claude 3.7 Sonnet', 'GPT 4.1', 'GPT o1', etc. without specific API versions or snapshot dates. Only Llama models have specific parameter counts."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided. The paper describes prompt structure at a high level but does not include the prompts used for evaluation."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states: 'For Llama 3.1 405B and 70B, we set the decoding parameters to T=0.2 and top-p=0.7. For the other models we used the default temperature and top-p supported by the API endpoint.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates agentic tasks in single-turn (non-agentic) format, noting 'no open-source, general-purpose hardware design agent currently exists.' No agentic scaffolding was used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the quality filtering pipeline: problems authored by experienced engineers, quality control via LLM judge scoring, automated filtering removing low-quality examples. Counts before/after filtering are given."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations' provides a dedicated discussion of benchmark limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: agentic context is oracle context (not realistic), Q&A tasks don't sufficiently challenge LLMs, tasks limited to standard hardware design, and commercial tool requirements for some categories."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6: 'the tasks in the benchmark are limited to standard hardware design and verification tasks and do not encompass the full range of challenges a design or verification engineer might face from project inception through fabrication.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (individual model outputs, per-problem results) is released. Only aggregate pass rates are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes how problems were authored by experienced hardware engineers across 13 categories, with difficulty levels and topical diversity requirements."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The acknowledgments thank 'Turing for their collaboration in developing the benchmark dataset' but do not describe how the hardware engineers who authored problems were recruited or selected."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from problem authoring through quality filtering is described: problems authored per category, quality control via LLM judge, automated filtering. Table 1 shows problem counts. Pre/post-filtering pass rate differences are quantified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure. The acknowledgments mention Turing collaboration and Cadence tool licenses but no explicit funding source."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as NVIDIA employees with NVIDIA email addresses."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "NVIDIA, as the employer of all authors, has a commercial interest in hardware design automation. Their benchmark could influence perception of AI capabilities in their domain. No independence statement is made."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Only GPT 4.1's knowledge cutoff (June 2024) is mentioned via the reference URL. No training cutoffs are stated for other models, and no systematic discussion of cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark problems or similar Verilog code appeared in any model's training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss contamination risk. Since this is a new benchmark, contamination risk is lower, but the paper does not explicitly address this."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Problems were authored by engineers but no human subjects research was conducted."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study requiring IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or latency figures reported despite evaluating 7 models across 783 problems with n=5 samples each."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, GPU hours, or API spend reported."
       }
     }
   },
   "claims": [
     {
       "claim": "State-of-the-art models achieve no more than 34% pass@1 on CVDP code generation tasks.",
       "evidence": "Table 2 shows Claude 3.7 Sonnet at 33.56% overall pass@1, the highest among all models tested.",
       "supported": "strong"
     },
     {
       "claim": "CVDP is substantially more challenging than prior Verilog benchmarks like VerilogEval v2.",
       "evidence": "Section 4 compares: VerilogEval v2 reported 57% for LLaMA 3.1 405B and 63% for GPT-4o, while CVDP yields 23% and 29% respectively for comparable models.",
       "supported": "strong"
     },
     {
       "claim": "Agentic tasks are more difficult than non-agentic tasks when evaluated in single-turn format.",
       "evidence": "Tables 2-3 show GPT 4.1 drops from 29% to 21% and Claude 3.7 Sonnet drops from 34% to 29% between non-agentic and agentic problems.",
       "supported": "moderate"
     },
     {
       "claim": "Design verification tasks (testbench/assertion generation) exhibit substantially lower pass rates than RTL generation.",
       "evidence": "Table 2 shows cid13 (testbench checker gen) at 3-10% across models vs cid02-04 (RTL gen) at 18-48%.",
       "supported": "strong"
     },
     {
       "claim": "Extended thinking mode provides minimal benefit for Claude 3.7 Sonnet on these tasks.",
       "evidence": "Tables 2-3 show Claude 3.7 Sonnet with and without thinking achieve nearly identical overall pass rates (33.56% vs 33.04% non-agentic, 29% vs 29% agentic).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The CVDP benchmark introduces 783 expert-authored Verilog design problems across 13 categories, substantially more challenging than prior benchmarks. State-of-the-art LLMs achieve at most 34% pass@1 on code generation, with design verification tasks (testbench and assertion generation) proving especially difficult at single-digit pass rates. The failure analysis reveals that verification tasks produce more diverse failure clusters than RTL generation tasks, suggesting deeper capability gaps in procedural/imperative code generation.",
   "red_flags": [
     {
       "flag": "Company evaluating domain relevance",
       "detail": "All 8 authors are NVIDIA employees. NVIDIA has commercial interest in hardware design automation. The benchmark could shape perception of AI readiness for EDA workflows in ways that affect NVIDIA's business strategy."
     },
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "Despite using n=5 samples per problem, no confidence intervals, error bars, or variance measures are reported. Differences between models (e.g., 34% vs 29%) may not be statistically significant."
     },
     {
       "flag": "Benchmark and code not publicly released",
       "detail": "The paper describes a benchmark but provides no download link, repository URL, or archive. Results cannot be independently verified or reproduced."
     },
     {
       "flag": "LLM-as-judge without validation",
       "detail": "GPT o4-mini is used as the scoring model for Q&A tasks, but no validation of this LLM judge against human ratings is provided. The paper acknowledges 'further investigation is needed to assess the technical reliability of these scores.'"
     }
   ],
   "cited_papers": [
     {
       "title": "VerilogEval: Evaluating large language models for Verilog code generation",
       "authors": ["Mingjie Liu", "Nathaniel Pinckney", "Brucek Khailany", "Haoxing Ren"],
       "year": 2023,
       "relevance": "Prior Verilog code generation benchmark that CVDP extends and supersedes."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "Influential software engineering benchmark for LLM agents; CVDP is analogous for hardware design."
     },
     {
       "title": "CraftRTL: High-quality synthetic data generation for verilog code models with correct-by-construction non-textual representations and targeted code repair",
       "authors": ["Mingjie Liu", "Yun-Da Tsai", "Wenfei Zhou", "Haoxing Ren"],
       "year": 2025,
       "relevance": "Synthetic data generation for Verilog LLMs, directly related to improving model capability on hardware code generation."
     },
     {
       "title": "VerilogCoder: Autonomous verilog coding agents with graph-based planning and abstract syntax tree (AST)-based waveform tracing tool",
       "authors": ["Chia-Tung Ho", "Haoxing Ren", "Brucek Khailany"],
       "year": 2024,
       "arxiv_id": "2408.08927",
       "relevance": "Agentic Verilog coding system directly relevant to the agentic evaluation format in CVDP."
     },
     {
       "title": "Copilot evaluation harness: Evaluating LLM-guided software programming",
       "authors": ["Anisha Agarwal", "Aaron Chan", "Shubham Chandel"],
       "year": 2024,
       "arxiv_id": "2402.14261",
       "relevance": "LLM evaluation framework for code generation, methodologically related to CVDP's evaluation infrastructure."
     },
     {
       "title": "RTL-Repo: A benchmark for evaluating LLMs on large-scale RTL design projects",
       "authors": ["Ahmed Allam", "Mohamed Shalan"],
       "year": 2024,
       "relevance": "Alternative RTL benchmark focusing on large-scale design projects, complementary to CVDP."
     },
     {
       "title": "Large language model for verilog generation with code-structure-guided reinforcement learning",
       "authors": ["Ning Wang", "Bingkun Yao", "Jie Zhou", "Xi Wang", "Zhe Jiang", "Nan Guan"],
       "year": 2025,
       "arxiv_id": "2407.18271",
       "relevance": "RL-based approach to improving LLM Verilog generation, directly evaluated on similar tasks."
     },
     {
       "title": "LLM4DV: Using large language models for hardware test stimuli generation",
       "authors": ["Zixi Zhang", "Greg Chadwick", "Hugo McNally", "Yiren Zhao", "Robert Mullins"],
       "year": 2023,
       "arxiv_id": "2310.04535",
       "relevance": "LLM-based hardware test generation, directly relevant to CVDP's verification task categories."
     }
   ]
 }