ai-research-survey

scan.json (28498B)

---

 {
   "paper": {
     "title": "OpenLLM-RTL: Open Dataset and Benchmark for LLM-Aided Design RTL Generation",
     "authors": [
       "Shang Liu",
       "Yao Lu",
       "Wenji Fang",
       "Mengming Li",
       "Zhiyao Xie"
     ],
     "year": 2024,
     "venue": "ICCAD '24 (ACM/IEEE International Conference on Computer-Aided Design)",
     "arxiv_id": "2503.15112",
     "doi": "10.1145/3676536.3697118"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "OpenLLM-RTL integrates three open-source contributions: RTLLM 2.0 (50 hand-crafted RTL designs for benchmarking), AssertEval (18 designs for assertion-based verification evaluation), and RTLCoder-Data (80K raw + 7K verified instruction-code samples). A DeepSeek-Coder-6.7B model fine-tuned on the 80K dataset outperforms GPT-3.5 and matches GPT-4 on VerilogEval. Critically, 7K verified samples outperform 27K-50K raw samples on most metrics, demonstrating that assertion-based data quality verification substantially improves training efficiency.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Three GitHub repositories are provided: RTLLM 2.0 at github.com/hkust-zhiyao/RTLLM, AssertEval at github.com/hkust-zhiyao/AssertLLM, and RTLCoder-Data at github.com/hkust-zhiyao/RTL-Coder (Section 2, 3, 4 footnotes)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All three benchmarks and datasets are open-sourced: RTLLM 2.0 (50 designs with descriptions, testbenches, and reference RTL), AssertEval (18 designs with specifications and golden RTL), and RTLCoder-Data (80K raw + 7K verified samples)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware (4x RTX 4090 24GB) and DeepSpeed stage-2, but provides no requirements.txt, Dockerfile, or detailed dependency listing with library versions. Software environment is not specified beyond framework names."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology is described at a high level (Section 4.1, 5.1) but there are no runnable commands, scripts, or README-level instructions for reproducing the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 5 are point estimates (e.g., '64.7%' pass@1) with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'outperforms GPT4 by an absolute value of 4.7%' (Section 5.3) are made by comparing point estimates with no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute and relative differences are reported with baseline context, e.g., 'performance on the Eval-Machine pass@1 metric rises from 53.7% to 64.7%' as data increases from 5K to 80K (Section 5.3). Table 5 provides both system and baseline scores."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for benchmark sizes (50 designs in RTLLM 2.0, 18 in AssertEval). The expansion from 30 to 50 designs is motivated by 'more thorough evaluation' but no quantitative rationale is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported across experimental runs. Three temperature conditions are tested and only the best is reported: 'we evaluate all 3 temperature conditions {0.2, 0.5, 0.8} and report the best performance for each model' (Section 5.1)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 includes extensive baselines: commercial (GPT-3.5, GPT-4), closed-source (ChipNeMo, VerilogEval, BetterV), and open-source (Codegen2, Starcoder, Thakur et al., Mistral-7B, DeepSeek-Coder)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include 2024 models: BetterV, DeepSeek-Coder, GPT-4. These are contemporary and competitive at the time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 systematically studies impact of training data amount (5K/27K/50K/80K), training scheme (direct vs scoring-based), and data quality (raw vs verified 7K). These are ablation-style studies isolating individual factors."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: pass@1, pass@5, pass@10 on VerilogEval (Eval-Machine and Eval-Human), and Syntax-VCS and Func on RTLLM V1.1 (Table 5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated: synthesis for syntax correctness, testbench simulation for functionality, and PPA measurement for design quality. No human evaluation of generated RTL quality is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "VerilogEval and RTLLM are used as independent test benchmarks. Training data overlap is checked via Rouge-L similarity (Figure 4a), and samples with Rouge-L > 0.5 are removed during training (Section 5.2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "RTLLM 2.0 categorizes designs into Arithmetic, Memory, Control, and Miscellaneous modules (Table 2). AssertEval categorizes by design type: cryptographic, processor, arithmetic, communication, memory (Table 3). However, results in Table 5 are aggregated, not broken down by category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or error analysis is presented. The paper does not discuss where models fail, what types of designs are hardest, or show example generation failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows improvement. No failed approaches, unsuccessful configurations, or negative results are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about benchmark sizes (50 designs, 18 designs, 80K/7K samples), the verification-based data quality method, and LLM performance improvement are all supported by results in Sections 2-5 and Table 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'enlarging the training dataset... significantly boost the model's code generation capabilities' are supported by controlled experiments varying data amount (5K→80K) while holding other factors constant (same base model, same training procedure). The study design with single-variable manipulation is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims general 'LLM-Aided Design RTL Generation' but all experiments are on Verilog only. The paper mentions VHDL and Chisel support (Section 2.1) but provides no non-Verilog results. Generalization to other HDLs is not bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed results. For example, the 7K verified dataset's superiority could partly reflect bias toward simpler designs (verification is easier for simple circuits), but this is not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3.2 explicitly distinguishes three progressive goals: syntax, functionality, and quality (PPA). The paper acknowledges that 'While the testbench samples a reasonable number of cases, passing them doesn't guarantee 100% functionality correctness,' recognizing the proxy gap between benchmark pass rates and actual design correctness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Fine-tuned models specify versions (Mistral-7B-v0.1, DeepSeek-Coder-6.7b-Instruct), but commercial baselines are listed only as 'GPT-3.5' and 'GPT4' without API versions or snapshot dates (Table 5). Model behavior varies across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The data generation flow (Figure 3) describes prompt types at a high level (keyword-based, code-based, mutation) but actual prompt text is not provided. Evaluation prompts (design descriptions) are provided as part of the benchmark, but the training data generation prompts are described only conceptually."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 reports: Adam optimizer with β1=0.9, β2=0.999, lr=1e-5, no weight decay, context length 2048, global batch size 256, DeepSpeed stage-2. Inference temperatures {0.2, 0.5, 0.8} tested."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are directly prompted or fine-tuned for single-turn generation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the 3-stage data generation pipeline in detail (Figure 3): keyword preparation, instruction generation with mutation and checking, reference code generation. Section 5.2 documents Rouge-L filtering (>0.5 removed). Diversity analysis is provided (Table 4)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitation and Challenges' provides substantive discussion of challenges in both RTL generation benchmarking and assertion generation, spanning design complexity, description detail, data leakage, and assertion quality."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: (1) overly complex designs fail to differentiate models, (2) description detail level affects whether the task is generation or translation, (3) Rouge-L 'may not be perfect' for leakage detection, (4) pre-training leakage is uncontrollable, (5) assertion quality depends on specification richness."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to Verilog-only, does not state what design complexity levels are excluded, and does not specify what types of RTL generation tasks are outside scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All datasets and benchmarks are open-sourced via GitHub (RTLLM 2.0, AssertEval, RTLCoder-Data with both 80K raw and 7K verified samples). Raw training data is available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the 3-stage automated data generation flow in detail: keyword preparation from GPT, instruction generation with mutation, and reference code generation. The process is clearly documented with Figure 3."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is synthetically generated via LLM prompting and collected from public Verilog sources. Benchmarks are hand-crafted by the authors."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 illustrates the full pipeline from keywords to final datasets. Section 4.2 explains the 80K raw dataset generation (relaxed diversity checking, enlarged source pool). Section 4.3 explains the 7K verified dataset (syntax + functionality checking). Token distributions shown in Figure 4b."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with Hong Kong University of Science and Technology (HKUST), clearly stated in the paper header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The paper evaluates the authors' own benchmarks and datasets (RTLLM, RTLCoder), creating a potential self-evaluation conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any model. GPT-3.5, GPT-4, Mistral, DeepSeek-Coder, and other baselines are used without specifying when their pre-training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 computes Rouge-L similarity between training data and test benchmarks (Figure 4a), finding most samples have low overlap (~0.25). Samples with Rouge-L > 0.5 are removed. However, this only covers fine-tuning data, not pre-training overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 acknowledges that 'leakage during the LLM pre-training process is difficult to control' and calls it 'a challenging open problem,' but no concrete measures are taken to address whether VerilogEval or RTLLM benchmarks appeared in pre-training data of commercial models."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency metrics are reported for any evaluated model."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (4x RTX 4090 GPUs) but total training time, GPU hours, or computational budget are not quantified. Only relative comparisons are given: '7K verified uses <20% of the training time of 50K' (Section 5.3)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity analysis. Results appear to be from single training runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of training runs is not stated. For RTLLM evaluation, 5 trials (pass@5) are used, but it is unclear how many independent training runs produced each model."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Three temperature settings {0.2, 0.5, 0.8} are tested at inference time with best reported, but no hyperparameter search budget for training is described. The choice of learning rate, batch size, etc. appears fixed without justification."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 states 'we evaluate all 3 temperature conditions {0.2, 0.5, 0.8} and report the best performance for each model.' The best temperature per model is selected without justification for why or showing all configurations' results."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate models on their own benchmarks (RTLLM, RTLLM 2.0) and compare against their own prior work (RTLCoder). This self-evaluation bias is not acknowledged or discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of different sizes (6.7B-16B) and different training compute are compared without matching or reporting compute budgets. The relative training efficiency of 7K verified vs 50K raw is noted qualitatively but not quantified."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "Section 2.3.2 defines syntax, functionality, and quality goals, but does not discuss whether pass@k on 50 hand-crafted designs measures actual RTL generation capability. Section 6 raises questions about design complexity and description detail without resolving them."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated via direct generation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal aspects. VerilogEval was published in 2023; models trained after 2023 may have seen its solutions. This is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The design descriptions include module names and I/O signal specifications, which could match training data patterns."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 computes Rouge-L similarity between training samples and benchmark test cases (Figure 4a), finding most have low similarity (~0.25). Samples with Rouge-L > 0.5 are filtered out to reduce overlap."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Rouge-L similarity calculation is used as a concrete leakage detection method (Section 5.2, Figure 4a). Training samples with Rouge-L > 0.5 against benchmark test cases are removed."
       }
     }
   },
   "claims": [
     {
       "claim": "DeepSeek-Direct (80K) outperforms GPT-3.5 and all non-commercial baselines on all metrics, and outperforms GPT-4 on VerilogEval Eval-Machine pass@1 by 4.7%",
       "evidence": "Table 5 shows DeepSeek-Direct (80K) achieves 64.7% pass@1 on Eval-Machine vs GPT-4's 60.0%. It exceeds GPT-3.5 (46.7%) and all open-source baselines across metrics.",
       "supported": "moderate"
     },
     {
       "claim": "Larger training datasets consistently improve model performance with no saturation at 80K samples",
       "evidence": "Table 5 and Figure 5 show monotonic improvement from 5K (53.7%) to 27K (59.8%) to 50K (62.6%) to 80K (64.7%) on Eval-Machine pass@1. Section 5.3 states 'even with 80K data samples, there are still no signs of model performance saturation.'",
       "supported": "moderate"
     },
     {
       "claim": "7K verified dataset outperforms 27K raw dataset on all metrics and 50K raw dataset on 6/8 metrics",
       "evidence": "Table 5 shows DeepSeek-Direct (7K verified) at 61.3% pass@1 vs DeepSeek-Direct (27K) at 59.8% and DeepSeek-Direct (50K) at 62.6% on Eval-Machine. On Eval-Human pass@5/10 and RTLLM Syntax, 7K verified exceeds 50K raw.",
       "supported": "moderate"
     },
     {
       "claim": "Scoring-based training scheme improves over basic direct training",
       "evidence": "Table 5 shows Mistral-Scoring and DeepSeek-Scoring outperform their Direct counterparts on all metrics with the same 27K data. E.g., DeepSeek-Scoring 61.2% vs DeepSeek-Direct 59.8% on Eval-Machine pass@1.",
       "supported": "moderate"
     },
     {
       "claim": "RTLCoder-Data exhibits satisfactory diversity compared with other open-source Verilog datasets",
       "evidence": "Table 4 shows lower CR (4.21) and CR:POS (7.33) for RTLCoder-Data Raw compared to MG-Verilog (5.80/9.16) and Goh et al. (5.27/10.1), where lower values indicate higher diversity.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Best-of-temperatures reporting inflates results",
       "detail": "Section 5.1 states 'we evaluate all 3 temperature conditions {0.2, 0.5, 0.8} and report the best performance for each model.' This cherry-picks the best configuration without showing variance or all results, inflating reported performance."
     },
     {
       "flag": "No error bars or variance on any result",
       "detail": "All results in Table 5 are single point estimates with no confidence intervals, standard deviations, or variance across runs. Given that LLM generation is stochastic, single-run results are unreliable."
     },
     {
       "flag": "Self-evaluation bias: authors benchmark on own dataset",
       "detail": "The authors evaluate their fine-tuned models on RTLLM, which they also authored (reference [29]). The benchmark design may favor their training data distribution. This conflict is not acknowledged."
     },
     {
       "flag": "Pre-training contamination unaddressed for baselines",
       "detail": "Commercial models (GPT-3.5, GPT-4) and open-source models may have seen VerilogEval or RTLLM benchmark problems during pre-training. Only fine-tuning overlap is checked via Rouge-L, not pre-training contamination."
     }
   ],
   "cited_papers": [
     {
       "title": "RTLCoder: Outperforming GPT-3.5 in Design RTL Generation with Our Open-Source Dataset and Lightweight Solution",
       "authors": ["Shang Liu", "Wenji Fang", "Yao Lu", "Qijun Zhang", "Hongce Zhang", "Zhiyao Xie"],
       "year": 2023,
       "arxiv_id": "2312.08617",
       "relevance": "Prior work proposing the RTLCoder dataset and scoring-based training scheme for LLM-based RTL code generation."
     },
     {
       "title": "VerilogEval: Evaluating Large Language Models for Verilog Code Generation",
       "authors": ["Mingjie Liu", "Nathaniel Pinckney", "Brucek Khailany", "Haoxing Ren"],
       "year": 2023,
       "arxiv_id": "2309.07544",
       "relevance": "Key benchmark for evaluating LLM Verilog generation capability, used as primary evaluation in this paper."
     },
     {
       "title": "ChipNeMo: Domain-Adapted LLMs for Chip Design",
       "authors": ["Mingjie Liu", "Teodor-Dumitru Ene", "Robert Kirby"],
       "year": 2023,
       "arxiv_id": "2311.00176",
       "relevance": "Domain-adapted LLM for chip design using closed-source training data, evaluated as baseline."
     },
     {
       "title": "BetterV: Controlled Verilog Generation with Discriminative Guidance",
       "authors": ["Zehua Pei", "Hui-Ling Zhen", "Mingxuan Yuan", "Yu Huang", "Bei Yu"],
       "year": 2024,
       "arxiv_id": "2402.03375",
       "relevance": "Contemporary LLM-based Verilog generation approach achieving strong results on VerilogEval."
     },
     {
       "title": "MG-Verilog: Multi-grained Dataset Towards Enhanced LLM-assisted Verilog Generation",
       "authors": ["Yongan Zhang", "Zhongzhi Yu"],
       "year": 2024,
       "arxiv_id": "2407.01910",
       "relevance": "Open-source multi-grained Verilog dataset for LLM training, compared for diversity metrics."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Base model used for fine-tuning in RTLCoder experiments, representative of code-specialized LLMs."
     },
     {
       "title": "AssertLLM: Generating and Evaluating Hardware Verification Assertions from Design Specifications via Multi-LLMs",
       "authors": ["Wenji Fang", "Mengming Li", "Min Li"],
       "year": 2024,
       "arxiv_id": "2402.00386",
       "relevance": "LLM-based assertion generation method whose techniques are used for the verification-based data quality checking in RTLCoder-Data."
     },
     {
       "title": "Benchmarking Large Language Models for Automated Verilog RTL Code Generation",
       "authors": ["Shailja Thakur", "Baleegh Ahmad", "Zhenxing Fan", "Hammond Pearce"],
       "year": 2023,
       "relevance": "Early benchmark paper for LLM Verilog generation, provides code-only open-source dataset used in data generation."
     },
     {
       "title": "CodeV: Empowering LLMs for Verilog Generation through Multi-Level Summarization",
       "authors": ["Yang Zhao", "Di Huang"],
       "year": 2024,
       "arxiv_id": "2407.10424",
       "relevance": "LLM approach for Verilog generation using multi-level code summarization with closed-source data."
     },
     {
       "title": "RTLLM: An Open-Source Benchmark for Design RTL Generation with Large Language Model",
       "authors": ["Yao Lu", "Shang Liu", "Qijun Zhang", "Zhiyao Xie"],
       "year": 2023,
       "arxiv_id": "2308.05345",
       "relevance": "Original RTLLM benchmark (30 designs) that this paper extends to RTLLM 2.0 (50 designs)."
     },
     {
       "title": "OriGen: Enhancing RTL Code Generation with Code-to-Code Augmentation and Self-Reflection",
       "authors": ["Fan Cui", "Chenyang Yin"],
       "year": 2024,
       "arxiv_id": "2407.16237",
       "relevance": "RTL code generation approach using data augmentation and self-reflection techniques."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundation model used as commercial baseline for RTL generation evaluation."
     }
   ]
 }