ai-research-survey

scan.json (27540B)

---

 {
   "paper": {
     "title": "RTL++: Graph-enhanced LLM for RTL Code Generation",
     "authors": [
       "Mohammad Akyash",
       "Kimia Azar",
       "Hadi Kamali"
     ],
     "year": 2025,
     "venue": "2025 IEEE International Conference on LLM-Aided Design (ICLAD)",
     "arxiv_id": "2505.13479",
     "doi": "10.1109/ICLAD65226.2025.00020"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RTL++ introduces a graph-augmented fine-tuning approach for LLM-based Verilog code generation, encoding RTL code as textualized control flow and data flow graphs alongside the code itself for instruction generation. On VerilogEval HumanEval, RTL++ (200K training set) achieves 59.9% pass@1, 68.8% pass@5, and 72.1% pass@10, outperforming GPT-4 and most baselines on pass@5/10, though it underperforms CraftRTL on pass@1. The ablation study shows graph-augmented instructions improve performance by ~5% at 5K scale, with claimed 18% improvement at 100K scale.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Footnote 1 links to a HuggingFace datasets URL (huggingface.co/datasets/makyash/RTL-PP) for the dataset/model, but no source code repository (training scripts, preprocessing code, evaluation pipeline) is provided. The conclusion also states 'we also plan to make RTL++ a fully open-source model,' indicating full release is future work."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 1 states 'Dataset/Model is available at [1]' with a HuggingFace URL (huggingface.co/datasets/makyash/RTL-PP). The evaluation uses public benchmarks VerilogEval and RTLLM."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section IV mentions PyTorch, NVIDIA L4, and LoRA, but no requirements.txt, Dockerfile, or detailed dependency listing with library versions is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts for replicating experiments are provided in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables III, IV, and V report only point estimates (e.g., 59.9% pass@1) with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'RTL++ outperforms state-of-the-art models' by comparing raw percentages across Tables III and IV without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables III and IV report absolute pass@k percentages with baselines providing context. For example, CodeLlama-7B at 18.2% pass@1 vs RTL++@200K at 59.9%, and GPT-4 at 43.5% vs RTL++@100K at 54.3%. Fig. 3 shows improvement rate per +1K training samples."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why the dataset sizes (5K to 200K) were chosen, no power analysis, and no discussion of whether the VerilogEval benchmark size is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures reported for any experimental results. All tables show single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table IV compares against seven baselines: CodeLlama-7B-Instruct, GPT-4, VeriGen, RTLCoder, BetterV, OriGen, AutoVCoder, CodeV, and CraftRTL."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent 2024 work: OriGen, BetterV, AutoVCoder, CodeV, and CraftRTL, all representing current state-of-the-art in RTL code generation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table V provides an ablation study comparing RTL++ with and without textualized graph representations (TGR) at 5K dataset size, showing 1.3-5.1% improvement with TGR at different temperatures."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses multiple metrics: VerilogEval pass@1, pass@5, pass@10, plus RTLLM 1.1 synthesis rate and functional correctness."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code quality. All evaluation is automated via VerilogEval (pass@k via simulation) and RTLLM (synthesis + functional). Human expert review could have assessed code readability, maintainability, or design quality beyond pass/fail."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "VerilogEval and RTLLM 1.1 are external benchmarks separate from the training data, serving as held-out test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported only as aggregate pass@k on VerilogEval and synthesis/functional rates on RTLLM. No per-task, per-category, or per-difficulty breakdown is provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No error analysis, failure cases, or discussion of where RTL++ breaks down. Only the ALU case study (Section IV-E, Fig 4) shows a positive example."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows consistent improvement. No mention of approaches tried and abandoned, configurations that failed, or ablations that hurt performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims RTL++ 'outperforms state-of-the-art models fine-tuned for RTL generation.' However, Table IV shows CraftRTL beats RTL++@200K on pass@1 (63.1 vs 59.9) and RTLLM functional (52.9 vs 51.7). RTL++ only leads on pass@5 (68.8 vs 67.8) and pass@10 (72.1 vs 69.7). The unqualified 'outperforms' claim is not fully supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim — that graph-based augmentation improves performance — is supported by the Table V ablation study which isolates the graph representation variable while controlling dataset size and model. This is a controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'RTL Code Generation' broadly but results are only on Verilog (no VHDL, SystemVerilog). Only CodeLlama-7B is used as the base model. The paper acknowledges DeepSeek could yield 'superior outcomes' (footnote 10) but doesn't bound claims to the tested configuration."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The improvement could be partly due to larger/different training data rather than graph representations specifically. The ablation at 5K doesn't rule out confounds at 200K scale."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures pass@k on VerilogEval and synthesis/functional rates on RTLLM, but frames results as 'advancing the capabilities of LLM-assisted RTL code generation' and 'high-quality' code generation. No discussion of the gap between benchmark pass rates and real-world RTL design quality (power, performance, area, maintainability)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "CodeLlama-7B-Instruct is specified as the base model (specific enough). However, GPT-4 and GPT-4o are used for data generation and refinement without version numbers or snapshot dates. Section IV says 'GPT-4 has been engaged' without specifying gpt-4-0613 or similar."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section III-B describes the GPT refinement prompt in natural language ('guided GPT through several key steps') and Section III-D describes instruction generation prompting, but no actual prompt text is provided anywhere in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV reports: learning rate 2e-4, AdamW optimizer with β1=0.9, β2=0.99, cosine decay schedule, warmup ratio 0.03, batch size 2, 1 epoch, LoRA technique. Table V reports temperature settings (0.6, 0.7)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. RTL++ is a fine-tuned LLM evaluated on standard benchmarks with direct generation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A documents data collection (keyword-based search, 100-300 line filtering, star-count ranking, testbench/netlist exclusion). Section III-B documents GPT-based refinement (dependency removal, variable initialization, syntax correction, Yosys synthesis validation)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has no limitations, threats-to-validity, or discussion section. Structure is: Introduction, Related Work, Proposed Model, Experiments, Conclusion — with no limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries stated. The paper does not acknowledge that results are limited to Verilog only, CodeLlama-7B only, or that VerilogEval HumanEval subset may not represent all RTL design scenarios."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The training dataset is linked on HuggingFace, but raw experimental outputs (generated code samples, per-problem pass/fail results, benchmark logs) are not available for independent verification of the reported numbers."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes the collection procedure in detail: keyword-based search from GitHub, Bitbucket, and Opencores; star-count ranking; line-count filtering (100-300 lines); testbench/netlist exclusion; hierarchical module inclusion."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is collected from public code repositories (GitHub, Bitbucket, Opencores)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline stages are described (collection → refinement → graph generation → instruction generation → fine-tuning) but intermediate counts are missing. The paper does not state how many samples were initially collected, how many were excluded at each filtering stage, or how many failed synthesis validation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations with University of Central Florida, Department of ECE, are clearly listed with email addresses. They are not evaluating a commercial product they are affiliated with."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The paper mentions ~$84 per 1000 samples for GPT-4 usage, indicating non-trivial costs, but the funding source is unstated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for CodeLlama-7B-Instruct. The custom training data is described as collected from GitHub, Bitbucket, and Opencores but no temporal cutoff is given."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between the 200K training samples (collected from GitHub) and the VerilogEval benchmark (also derived from GitHub/open-source Verilog). This is a significant concern since both draw from the same ecosystem."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "VerilogEval was published in 2023 and its problems are publicly available. CodeLlama was also trained on public code. No discussion of whether the benchmark solutions or problems appeared in either the base model's training data or RTL++'s custom training set."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Evaluation is entirely automated via benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed is reported for the fine-tuned model. Only data generation cost (~$84 per 1000 samples for GPT-4) is mentioned."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (NVIDIA L4) and GPT-4 data generation cost (~$84/1K samples), but total GPU hours for fine-tuning, total training time, and total project compute cost are not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Results appear to be from a single training run per configuration."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without indicating whether they are from single or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters are reported but no search budget or method is described. No indication of how many configurations were tried before selecting the reported settings."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explain how the final hyperparameter configuration was selected or whether a validation set was used for selection."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 9+ models and multiple metrics without any statistical tests or correction procedures."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system against baselines without acknowledging evaluation bias. Some baselines use numbers from original papers while RTL++ numbers are self-reported."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "RTL++@200K uses 200K training samples vs CraftRTL's 80.1K and RTLCoder's 27K. The performance comparison does not normalize for dataset size or compute. Fig 3 shows data-size vs performance but doesn't compare baselines at matched compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether VerilogEval or RTLLM actually measure RTL code generation quality in a meaningful way. The paper uses VerilogEval's HumanEval subset based on a brief citation ([40]) but does not analyze construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. Models are evaluated via direct code generation on benchmarks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The training data is collected from public repositories without temporal restrictions, and VerilogEval problems have been publicly available since 2023."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage or whether the evaluation setup provides information not available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether training data (from GitHub, Bitbucket, Opencores) and VerilogEval/RTLLM test problems are independent. Both draw from the open-source Verilog ecosystem, creating a serious overlap risk."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No deduplication against benchmarks, no canary strings, no membership inference testing."
       }
     }
   },
   "claims": [
     {
       "claim": "RTL++ with 200K training samples achieves 59.9% pass@1, 68.8% pass@5, 72.1% pass@10 on VerilogEval HumanEval subset.",
       "evidence": "Table III and Table IV report these exact numbers for RTL++@200K on VerilogEval HumanEval.",
       "supported": "moderate"
     },
     {
       "claim": "RTL++ outperforms GPT-4 on VerilogEval when trained on 100K+ dataset.",
       "evidence": "Table III shows RTL++@100K at 54.3% pass@1 vs GPT-4 at 43.5% pass@1. No significance testing.",
       "supported": "moderate"
     },
     {
       "claim": "RTL++ outperforms state-of-the-art fine-tuned models for RTL generation.",
       "evidence": "Table IV shows RTL++@200K beats CraftRTL on pass@5 (68.8 vs 67.8) and pass@10 (72.1 vs 69.7), but loses on pass@1 (59.9 vs 63.1) and RTLLM functional correctness (51.7 vs 52.9). The claim is only partially supported.",
       "supported": "weak"
     },
     {
       "claim": "Graph-augmented instruction generation improves fine-tuning performance by up to 5% at 5K scale and 18% at 100K scale.",
       "evidence": "Table V shows a 5.1% improvement at 5K (25.6 → 30.7 pass@10 at temp 0.7). The 18% improvement at 100K is stated in Section IV-D text but not shown in a table.",
       "supported": "weak"
     },
     {
       "claim": "Dataset size consistently improves model quality with diminishing marginal returns.",
       "evidence": "Table III and Fig. 3 show a clear scaling trend from 5K to 200K training samples, with diminishing improvement rate shown in the secondary axis of Fig. 3.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Abstract overclaims relative to results",
       "detail": "The abstract claims RTL++ 'outperforms state-of-the-art models' but Table IV shows CraftRTL beats RTL++@200K on pass@1 (63.1 vs 59.9) and RTLLM functional correctness (52.9 vs 51.7). The claim holds only for pass@5 and pass@10."
     },
     {
       "flag": "No error bars or variance on any results",
       "detail": "All results across Tables III, IV, and V are single point estimates with no indication of variance across seeds or runs. Pass@k metrics are inherently noisy and a single run may not be representative."
     },
     {
       "flag": "Training/test data contamination risk",
       "detail": "Training data collected from GitHub, Bitbucket, and Opencores. VerilogEval is also derived from open-source Verilog. No overlap analysis or deduplication is performed or discussed."
     },
     {
       "flag": "Unfair compute comparison",
       "detail": "RTL++@200K uses 200K training samples while CraftRTL uses 80.1K, RTLCoder uses 27K, and CodeV uses 165K. Performance is not normalized for training data size or compute budget, making comparisons misleading."
     },
     {
       "flag": "Ablation only at smallest scale",
       "detail": "The graph augmentation ablation (Table V) is only shown at the 5K dataset scale. The claimed 18% improvement at 100K is mentioned in text without supporting table data, making the key contribution's impact at scale unverifiable."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper entirely lacks a limitations or threats-to-validity section despite significant limitations: single base model, Verilog only, no variance reporting, potential data overlap."
     }
   ],
   "cited_papers": [
     {
       "title": "VeriGen: A Large Language Model for Verilog Code Generation",
       "authors": ["S. Thakur"],
       "year": 2023,
       "arxiv_id": "2308.00708",
       "relevance": "Early fine-tuning approach for Verilog code generation from GitHub and textbook data, key baseline."
     },
     {
       "title": "RTLCoder: Outperforming GPT-3.5 in Design RTL Generation with Our Open-Source Dataset and Lightweight Solution",
       "authors": ["S. Liu"],
       "year": 2024,
       "relevance": "GPT-3.5-based synthetic data approach for RTL code generation, competing method with 27K training pairs."
     },
     {
       "title": "OriGen: Enhancing RTL Code Generation with Code-to-Code Augmentation and Self-Reflection",
       "authors": ["F. Cui"],
       "year": 2024,
       "arxiv_id": "2407.16237",
       "relevance": "Code-to-code augmentation and self-reflection for Verilog generation, strong baseline with 222K training samples."
     },
     {
       "title": "CraftRTL: High-quality Synthetic Data Generation for Verilog Code Models with Correct-by-Construction Non-Textual Representations and Targeted Code Repair",
       "authors": ["M. Liu"],
       "year": 2024,
       "arxiv_id": "2409.12993",
       "relevance": "Correct-by-construction data approach for RTL LLMs, strongest baseline in the comparison (63.1% pass@1)."
     },
     {
       "title": "VerilogEval: Evaluating Large Language Models for Verilog Code Generation",
       "authors": ["M. Liu"],
       "year": 2023,
       "relevance": "Primary benchmark used for evaluation, standard pass@k metric for Verilog code generation."
     },
     {
       "title": "AutoVCoder: A Systematic Framework for Automated Verilog Code Generation Using LLMs",
       "authors": ["M. Gao"],
       "year": 2024,
       "relevance": "Domain-specific RAG approach for Verilog generation, competing method incorporating retrieval augmentation."
     },
     {
       "title": "CodeV: Empowering LLMs for Verilog Generation through Multi-Level Summarization",
       "authors": ["Y. Zhao"],
       "year": 2024,
       "arxiv_id": "2407.10424",
       "relevance": "Multi-level code summarization approach for Verilog, shifts focus from generation to description-code pairs."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["B. Rozière"],
       "year": 2024,
       "arxiv_id": "2308.12950",
       "relevance": "Base model used for RTL++ fine-tuning, widely used open-source code LLM."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming – The Rise of Code Intelligence",
       "authors": ["D. Guo"],
       "year": 2024,
       "relevance": "Alternative base model for code generation; several baselines in Table IV use DeepSeek-Coder variants."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["E. J. Hu"],
       "year": 2021,
       "arxiv_id": "2106.09685",
       "relevance": "Parameter-efficient fine-tuning technique used by RTL++ for adapting CodeLlama to RTL generation."
     },
     {
       "title": "RTLLM: An Open-Source Benchmark for Design RTL Generation with Large Language Model",
       "authors": ["Y. Lu"],
       "year": 2023,
       "arxiv_id": "2308.05345",
       "relevance": "Secondary benchmark used for evaluation, measuring synthesis rate and functional correctness of generated RTL."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex paper establishing pass@k evaluation methodology widely used in code generation research."
     },
     {
       "title": "BetterV: Controlled Verilog Generation with Discriminative Guidance",
       "authors": ["Z. Pei"],
       "year": 2024,
       "arxiv_id": "2402.03375",
       "relevance": "Controlled text generation framework for Verilog with PPA optimization, competing approach."
     }
   ]
 }