ai-research-survey

scan.json (26396B)

---

 {
   "paper": {
     "title": "AutoVCoder: A Systematic Framework for Automated Verilog Code Generation using LLMs",
     "authors": [
       "Mingzhe Gao",
       "Jieru Zhao",
       "Zhe Lin",
       "Wenchao Ding",
       "Xiaofeng Hou",
       "Yu Feng",
       "Chao Li",
       "Minyi Guo"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2407.18333"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes AutoVCoder as 'open-source' in the abstract and introduction, but no GitHub link, Zenodo archive, or any repository URL is provided anywhere in the paper. A claim of being open-source without a working URL does not satisfy this criterion."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes constructing a high-quality hardware dataset from GitHub (~1M modules, ~50k after filtering) and a synthetic dataset, but no download links or data releases are provided. The benchmarks used (VerilogEval, RTLLM) are publicly available, but the paper's own generated datasets are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'three Nvidia A100 GPUs' (Section IV-A) and names base models (Codellama-7B, DeepSeek-Coder-6.7B, CodeQwen1.5-7B) but provides no requirements.txt, Dockerfile, or detailed environment setup listing library versions. This is insufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README with commands, or scripts are provided. The methodology section describes the approach conceptually but does not include specific commands or procedures for replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables I, II, and III report only point estimates (e.g., '69.0%', '79.3%') with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes multiple comparative claims (e.g., 'AutoVCoder outperforms both industrial and academic LLMs') based solely on comparing raw percentages in Tables I-III without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context: '0.5% and 2.2% improvement in functional correctness on the EvalMachine and EvalHuman benchmarks compared with BetterV' and '3.4% increase in syntax correctness and a 3.4% increase in functional correctness on the RTLLM benchmark compared with RTLCoder.' Tables provide both baseline and system scores allowing readers to compute absolute differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses n=10 for pass@k evaluation without justifying why 10 samples are sufficient. No power analysis or discussion of whether the benchmarks (100+ tasks for VerilogEval, 29 tasks for RTLLM) provide adequate statistical power for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported across experimental runs. The pass@k metric is computed from n=10 generations but no variance across seeds or repeated experiments is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table I includes extensive baselines: GPT-3.5, GPT-4, ChipNeMo, VerilogEval, Codegen2, Starcoder, Thakur et al., RTLCoder (Mistral and DeepSeek variants), and BetterV (Codellama, DeepSeek, CodeQwen variants)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include contemporary work: BetterV (ICML 2024), RTLCoder (2023), and GPT-4. These represent the state-of-the-art for Verilog code generation at the time of submission."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table II provides ablation of the two-round fine-tuning (no fine-tuning, round 1 only, round 2 only, both rounds). Table III provides ablation of the RAG components (no RAG, example retriever only, knowledge retriever only, both retrievers). These cover the three main components of the framework."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: pass@1 and pass@5 for VerilogEval, and syntactic correctness and functional correctness for RTLLM. This provides evaluation from multiple angles."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated Verilog code quality is included. All evaluation is automated via Icarus Verilog syntax checking and testbench execution. Human evaluation of code quality (readability, design patterns, synthesizability) would be relevant given claims about 'high-quality' code generation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses established external benchmarks (VerilogEval and RTLLM V1.1) as test sets, which are separate from the training data (GitHub open-source database and ChatGPT-generated synthetic dataset). This provides a clear separation between training and evaluation data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by benchmark (VerilogEval EvalMachine, VerilogEval EvalHuman, RTLLM), by base model (Codellama, DeepSeek, CodeQwen), and by metric (syntax correctness, functional correctness, pass@1, pass@5) across all tables."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure patterns: LLMs tend to 'use loops like those in Python and C++' leading to 'excessive usage of for loops when generating RTL code' (Section III-C). Section IV-D notes 'the example database is not well-matched to some of the difficult problems in RTLLM, due to the scarcity of large-scale Verilog design instances.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table III shows that the knowledge retriever alone sometimes does not improve or even slightly reduces performance (e.g., AutoVCoder-DeepSeek-KR has lower EvalMachine pass@5 than AutoVCoder-DeepSeek-1&2: 77.5% vs 77.8%). The paper also notes that 'improvements on RTLLM question set are less notable' (Section IV-D)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '0.5% and 2.2% improvement in functional correctness on the EvalMachine and EvalHuman benchmarks compared with BetterV, and also achieves a 3.4% increase in syntax correctness and a 3.4% increase in functional correctness on the RTLLM benchmark compared with RTLCoder.' These specific numbers are supported by Table I (comparing AutoVCoder-CodeQwen vs BetterV-CodeQwen for VerilogEval, and AutoVCoder-DeepSeek vs RTLCoder-DeepSeek for RTLLM)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about the contribution of each component (fine-tuning rounds, RAG retrievers). Tables II and III provide controlled ablation studies where single components are added or removed while keeping other variables constant, which is adequate for causal inference about component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to be 'a systematic framework for Automated Verilog Code Generation' broadly, but results are limited to two benchmarks (VerilogEval with ~100 tasks, RTLLM with 29 tasks) and three base models (all 7B-class). The title and abstract do not bound claims to these specific settings. No discussion of whether results would hold for larger models, different HDL languages, or industrial-scale designs."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements. For instance, it does not consider whether improvements could be due to data leakage between the GitHub-sourced training data and the benchmarks, or whether the gains are simply from having more training data rather than the specific methodology."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies base model names with sizes: 'Codellama-7B [25], DeepSeek-Coder-6.7B [26], and CodeQwen1.5-7B [27]' (Section IV-A). These are specific model identifiers that can be unambiguously located. ChatGPT-3.5 is used for data generation without a specific version, but it is not the system being evaluated."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides actual prompt text used for data generation: Fig. 3 shows the code scoring prompt, and Fig. 5 shows the problem-code pair generation prompt template with specific variable definitions ($level, $circuit_type, $problem_type). These are the actual prompts used in the pipeline."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A reports: learning rate γ = 2e-4 for LLM fine-tuning, learning rate 1e-5 for retriever training, 1 epoch for first-round fine-tuning, 3 epochs for second-round fine-tuning, top_p = 0.95, temperature = 0.8, n = 10 for pass@k, retriever chunk counts (2 for example retriever, 3 for knowledge retriever)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The RAG scaffolding is described in detail in Section III-C: the example retriever and knowledge retriever are explained, including how they fetch information, how retrieved chunks are combined with user prompts, the contrastive learning training process (Fig. 7), and the inference pipeline (Fig. 2)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes the data pipeline: collection from ~20,000 GitHub repositories yielding ~1M raw modules, scoring via ChatGPT-3.5 trained code scorer, filtering threshold of 6.5 (21.7% retained), and synthetic dataset generation with code filter verification (syntax check via Icarus Verilog, functional correctness check via Python equivalents for combinational circuits). Fig. 6 illustrates the flow."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The conclusion (Section V) does not discuss limitations either."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper briefly notes in Section IV-D that 'the example database is not well-matched to some of the difficult problems in RTLLM,' but this is a result observation, not a systematic discussion of validity threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what the results do NOT show — for example, no mention that results are limited to small benchmarks, 7B-class models only, or that the approach has not been validated on industrial-scale designs."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available for independent verification. The generated datasets (open-source database, synthetic dataset), model outputs, and per-task benchmark results are not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes data collection in detail: GitHub search for .v files across ~20,000 repositories yielding ~1M modules, ChatGPT-3.5 scoring of 15,000 modules for code scorer training, code scorer applied to remaining modules with 6.5 threshold (21.7% retained). Synthetic dataset generated via ChatGPT-3.5 with verification pipeline (Fig. 6)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved. The paper uses automated benchmarks and LLM-generated data."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented across Section III-A and Figs. 4 and 6: GitHub scraping → module segmentation → code scoring (ChatGPT-3.5 → code scorer model) → threshold filtering → first-round fine-tuning data. Separately: ChatGPT-3.5 generation → syntax verification (Icarus Verilog) → functional verification (Python equivalence or testbench) → second-round fine-tuning data. However, exact counts of the final synthetic dataset size are not provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned anywhere in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Shanghai Jiao Tong University, Sun Yat-sen University, and Fudan University. These are academic institutions, not companies whose products are being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of the funder cannot be assessed. The absence of any funding disclosure is itself a concern."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper fine-tunes models (Codellama-7B, DeepSeek-Coder-6.7B, CodeQwen1.5-7B) and evaluates them on VerilogEval and RTLLM benchmarks, but does not state the training data cutoff dates for these base models. It is possible that the base models' pre-training data included VerilogEval benchmark problems."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the base models' pre-training data or the GitHub-sourced fine-tuning data overlaps with the VerilogEval or RTLLM benchmark problems. The open-source database was scraped from GitHub, and VerilogEval benchmarks were also derived from public sources, creating contamination risk."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "VerilogEval was published in 2023, and the base models were trained on data that may include these benchmarks. The paper does not address this contamination risk at all. Additionally, the GitHub-scraped training data could overlap with benchmark problems."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of LLM-based Verilog code generation."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or token consumption is reported. The RAG pipeline involves retriever queries plus LLM inference, and the cost implications of this are not discussed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'three Nvidia A100 GPUs' (Section IV-A) but does not report total GPU hours, training time, API costs for ChatGPT-3.5 data generation (which involved scoring ~1M modules and generating synthetic datasets), or total computational budget."
       }
     }
   },
   "claims": [
     {
       "claim": "AutoVCoder outperforms both industrial and academic LLMs in Verilog code generation, showing 0.5% and 2.2% improvement in functional correctness on EvalMachine and EvalHuman compared with BetterV.",
       "evidence": "Table I shows AutoVCoder-CodeQwen achieves 79.9% and 55.9% pass@5 on EvalMachine and EvalHuman respectively, vs BetterV-CodeQwen at 79.4% and 53.7%. The differences are 0.5% and 2.2%.",
       "supported": "weak"
     },
     {
       "claim": "AutoVCoder achieves a 3.4% increase in syntax correctness and 3.4% increase in functional correctness on RTLLM compared with RTLCoder.",
       "evidence": "Table I shows AutoVCoder-DeepSeek achieves 100% syntax and 51.7% functional correctness on RTLLM, vs RTLCoder-DeepSeek at 93.1% syntax and 48.3% functional. The differences are 6.9% and 3.4%. The 3.4% syntax claim appears to compare against RTLCoder-Mistral (96.6%).",
       "supported": "moderate"
     },
     {
       "claim": "Both rounds of fine-tuning are crucial, with the second round having more significant impact on correctness improvement.",
       "evidence": "Table II shows consistent improvements across all three base models. For CodeQwen: base 52.8% EvalMachine pass@5, round-1 only 65.4%, round-2 only 75.1%, both rounds 78.3%. The second round alone provides larger gains than the first round alone.",
       "supported": "moderate"
     },
     {
       "claim": "The domain-specific RAG technique, combining example and knowledge retrievers, achieves the best performance in most cases.",
       "evidence": "Table III shows that ER&KR combined generally matches or exceeds individual retriever configurations. For AutoVCoder-CodeQwen: 1&2 baseline 78.3%, ER 79.1%, KR 79.3%, ER&KR 79.9% on EvalMachine pass@5.",
       "supported": "moderate"
     },
     {
       "claim": "AutoVCoder outperforms GPT-4 on the EvalMachine question set.",
       "evidence": "Table I shows AutoVCoder-CodeQwen achieves 79.9% pass@5 on EvalMachine vs GPT-4 at 70.6%. However, GPT-4 has no fine-tuning on Verilog-specific data, making this an unfair comparison of a specialized fine-tuned 7B model against a general-purpose model.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AutoVCoder proposes a three-component framework for Verilog code generation: high-quality dataset generation from GitHub with automated scoring, two-round LoRA fine-tuning (general Verilog syntax then task-specific problem-code pairs), and domain-specific RAG with example and knowledge retrievers. On VerilogEval and RTLLM benchmarks, AutoVCoder achieves marginal improvements over prior SOTA (0.5-3.4% gains), with ablation studies confirming each component contributes. The improvements are small and reported without statistical significance testing or variance analysis, making it difficult to assess whether the gains are robust.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims are based on raw percentage differences (0.5%-3.4%) without any significance tests. Given that pass@k is estimated from only n=10 generations and RTLLM has only 29 tasks, the reported differences could easily be within noise."
     },
     {
       "flag": "No variance or error bars reported",
       "detail": "Results are reported as single numbers with no standard deviation, confidence intervals, or repeated runs. For stochastic LLM outputs (temperature=0.8, top_p=0.95), variance could be substantial."
     },
     {
       "flag": "Potential training-test contamination",
       "detail": "The open-source database is scraped from ~20,000 GitHub repositories. VerilogEval benchmarks are also derived from public Verilog sources. No analysis of whether training data overlaps with benchmark problems. Additionally, base model pre-training data may include the benchmarks."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section, which is a significant omission for a paper making comparative performance claims."
     },
     {
       "flag": "Claimed 'open-source' but no code released",
       "detail": "The abstract and introduction describe AutoVCoder as 'open-source' but no repository URL, download link, or code archive is provided anywhere in the paper."
     },
     {
       "flag": "Very small improvement margins",
       "detail": "The headline improvements (0.5% on EvalMachine, 2.2% on EvalHuman vs BetterV) are extremely small and could be within measurement noise, especially with n=10 samples and no significance testing."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational work introducing the pass@k metric and HumanEval benchmark for code generation evaluation."
     },
     {
       "title": "Codegen: An open large language model for code with multi-turn program synthesis",
       "authors": ["E. Nijkamp"],
       "year": 2023,
       "relevance": "Open-source LLM for code generation, used as a baseline for comparison."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2024,
       "relevance": "Major commercial LLM used as an industrial baseline for Verilog code generation comparison."
     },
     {
       "title": "RTLLM: An open-source benchmark for design RTL generation with large language model",
       "authors": ["Y. Lu"],
       "year": 2024,
       "relevance": "One of two primary evaluation benchmarks used in this paper for RTL code generation."
     },
     {
       "title": "RTLCoder: Outperforming GPT-3.5 in design RTL generation with our open-source dataset and lightweight solution",
       "authors": ["S. Liu"],
       "year": 2023,
       "relevance": "Prior SOTA for Verilog code generation using synthetic data and fine-tuning, a key baseline."
     },
     {
       "title": "BetterV: Controlled Verilog generation with discriminative guidance",
       "authors": ["P. Zehua"],
       "year": 2024,
       "relevance": "Most recent SOTA baseline for Verilog code generation, published at ICML 2024."
     },
     {
       "title": "VerilogEval: Evaluating large language models for Verilog code generation",
       "authors": ["M. Liu", "N. Pinckney"],
       "year": 2023,
       "relevance": "Primary evaluation benchmark used in this paper, provides EvalMachine and EvalHuman task sets."
     },
     {
       "title": "ChipNeMo: Domain-adapted LLMs for chip design",
       "authors": ["M. Liu"],
       "year": 2023,
       "arxiv_id": "2311.00176",
       "relevance": "Nvidia's domain-adapted LLM for chip design using two-round fine-tuning, a key related work."
     },
     {
       "title": "Benchmarking large language models for automated Verilog RTL code generation",
       "authors": ["S. Thakur"],
       "year": 2023,
       "relevance": "Early benchmark evaluation of LLMs for Verilog generation, used as baseline."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["B. Roziere"],
       "year": 2023,
       "relevance": "One of three base models used in AutoVCoder experiments (Codellama-7B)."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming - the rise of code intelligence",
       "authors": ["D. Guo"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "One of three base models used in AutoVCoder experiments (DeepSeek-Coder-6.7B)."
     },
     {
       "title": "LoRA: Low-rank adaptation of large language models",
       "authors": ["E. J. Hu"],
       "year": 2022,
       "relevance": "Fine-tuning method used in AutoVCoder's two-round fine-tuning approach."
     }
   ]
 }