ai-research-survey

scan.json (26809B)

---

 {
   "paper": {
     "title": "RTLSquad: Multi-Agent Based Interpretable RTL Design",
     "authors": [
       "Wang Bowei",
       "Qi Xiong",
       "Zeqing Xiang",
       "Lei Wang",
       "Renzhi Chen"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2501.05470",
     "doi": "10.48550/arXiv.2501.05470"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "RTLSquad is a multi-agent LLM system for RTL code generation that divides the design process into exploration, implementation, and verification stages managed by specialized agent squads. On the RTLLM V2.0 benchmark, it improved Pass@1 by 10.4pp (LLaMA) and 11.2pp (Deepseek) over the Self-Planning baseline. The system matched or exceeded reference PPA in most tested designs but failed entirely on 2 of 11 designs shown. The paper's central contribution is providing 'decision interpretability' through inter-agent communication, though this is demonstrated only through case studies with no empirical evaluation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'The optimized dataset will be made publicly available' — a future promise. The base RTLLM V2.0 is public but their modified version with enhanced testbenches is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is provided. Only model names and temperature are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions or scripts are provided. The methodology section describes the system architecture but not how to run it."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables I and II report only point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RTLSquad 'excels' and 'demonstrates better RTL code generation capability' compared to SP, but no statistical tests accompany these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table II reports Pass@1 with baseline context (LLaMA-SP 47.6 vs LLaMA-RTLSquad 58.0, Deepseek-SP 60.8 vs Deepseek-RTLSquad 72.0). Table I provides absolute PPA metrics for both reference and RTLSquad implementations."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The RTLLM V2.0 dataset contains 50 designs but no justification is given for why this sample size is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. All results are single-run point estimates despite using temperature 0.8 (stochastic generation)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against the Self-Planning (SP) method for Pass@1 (Table II) and against reference implementations and Deepseek-SP for PPA metrics (Table I)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Self-Planning (Jiang et al., Aug 2023) is about 1.4 years old at time of submission. The backbone models (Deepseek-V2.5, LLaMA 3.1-70B) are contemporary. However, more relevant baselines like MCTS-based RTLRewriter (2024) were not compared against."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (exploration squad, implementation squad, verification squad) but no ablation study isolates the contribution of individual components. No experiments remove or modify stages to measure their impact."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Pass@1 for functional correctness (Table II) and three PPA metrics — power (µW), performance/critical path (ns), and area (µm²) — for quality assessment (Table I)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. The core claim about 'decision interpretability' being useful to hardware engineers is never validated by actual engineers. All evaluation is automated (compilation, functional verification, synthesis)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "RTLLM V2.0 is used directly. The hyperparameters α=2.375, β=0.375, η=0.4 are stated as 'typically set' without explanation of how they were chosen, raising the possibility of test-set tuning. No validation set is mentioned."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table I provides per-design breakdown of all three PPA metrics for 11 designs, rather than only aggregate numbers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Table I shows RTLSquad failing entirely on right_shifter and width_8to16 (marked '-') and producing worse metrics on pulse_detect, but these failures are not discussed or analyzed in the text."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table I transparently shows designs where RTLSquad failed ('-' entries for right_shifter and width_8to16) and cases with worse metrics (pulse_detect power and area). The text acknowledges 'matching or exceeding... in most cases (73.3%)', implicitly admitting 26.7% did not improve."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'Pass@1 performance improving by 7.2%' but Table II shows improvements of 10.4pp (LLaMA: 47.6→58.0) and 11.2pp (Deepseek: 60.8→72.0), averaging 10.8pp. The 7.2% figure cannot be derived from the reported results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RTLSquad's multi-agent approach 'enhances backbone model's ability' but provides no ablation to isolate the causal factor. The improvement could stem from iterative refinement, additional LLM calls, EDA tool feedback, or the specific agent roles — these confounds are not addressed."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract present the system as a general solution for 'Interpretable RTL Design' but results are only on RTLLM V2.0 with Deepseek-V2.5 and LLaMA 3.1-70B using a 40nm process library. No scope boundaries are stated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The improvement over SP could simply be due to more LLM calls, iterative tool feedback, or enhanced testbench debugging outputs rather than the multi-agent architecture itself."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper's core contribution is 'decision interpretability' — the ability for engineers to trust and understand generated results. This is demonstrated through case studies (Section IV) but never measured. No user study, interpretability metric, or engineer feedback validates whether the decision paths are actually interpretable or useful. The gap between demonstration and claimed practical value is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: 'Deepseek-V2.5' and 'LLaMA 3.1-70B' (Section V-A). These identify both version and model size."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Agent roles and behaviors are described in natural language (programmer, reviewer, observer, analyst, experts) but no actual prompt text or system instructions are provided. The reader cannot reconstruct the prompts used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is reported (0.8). Exploration point parameters are stated: α=2.375, β=0.375, η=0.4 (Section III-C)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent scaffolding is described in detail across Sections III-A through III-D with workflow diagrams (Figures 1-4), agent roles, communication mechanisms, iterative inner/outer loops, and exploration point calculations."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We optimized the testbench files in the dataset, adding more debugging outputs' but does not describe what specific modifications were made, how many files were changed, or what debugging outputs were added."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No limitations, threats to validity, or similar section exists. The paper moves directly from experiments (Section V) to conclusion (Section VI)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No scope boundaries are stated. The paper does not discuss what types of designs, models, or scenarios the approach may not work for, despite failing on 2 of 11 evaluated designs."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (EDA reports, agent logs, full benchmark outputs, generated code) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes using RTLLM V2.0 dataset with 50 designs (Section V-A), commercial EDA tools with a 40nm process library for synthesis, and evaluation metrics extracted from synthesis reports."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is a standard benchmark (RTLLM V2.0)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The high-level workflow (agent processing → EDA tool → synthesis reports) is shown in figures, but the detailed pipeline — how EDA reports are parsed, how agent outputs are assembled, how final metrics are extracted from synthesis logs — is not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. No acknowledgments section is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: National University of Defense Technology (NUDT), Qiyuan Lab, and Academy of Military Science. All are Chinese military/government research institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence of the funder cannot be assessed. Authors are from military research institutions whose interest in the outcome is unknown."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosure appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for either Deepseek-V2.5 or LLaMA 3.1-70B, despite using them to generate code evaluated on the RTLLM V2.0 benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether RTLLM V2.0 benchmark designs appeared in the training data of either model."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "RTLLM V2.0 is a public benchmark that may have been included in model training data. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API expense, token consumption, or wall-clock time is reported despite the system making many iterative LLM calls per design across multiple agent squads."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The total GPU hours, API costs, or hardware used are not mentioned."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0.8 (stochastic generation) but only single-run results are reported. No seed sensitivity analysis is performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs used to compute Pass@1 is not stated. It is unclear how many samples were generated per design."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The parameters α=2.375, β=0.375, η=0.4 are stated as 'typically set' with no explanation of how these values were determined or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for parameter selection. The values α=2.375, β=0.375, η=0.4 are presented without derivation or selection methodology."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system against baselines without acknowledging self-evaluation bias. The SP baseline is re-implemented by the authors."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "RTLSquad uses multiple iterative LLM calls across several agent squads per design, far exceeding the single-call SP baseline. This compute asymmetry is not discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "RTLLM V2.0 is used without discussion of whether its 50 designs are representative of real-world RTL design complexity or whether Pass@1 on this benchmark measures practical design capability."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "RTLSquad's multi-agent scaffold is compared against the simpler SP approach. The improvement is attributed to the multi-agent design but could stem from more LLM calls, iterative feedback, or enhanced testbench outputs. This confound is not addressed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether RTLLM V2.0 benchmark problems existed before the training data cutoff of the models used."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors 'optimized the testbench files, adding more debugging outputs' to help the implementation stage. This enhanced feedback could constitute feature leakage (providing hints not available in normal usage) but is not discussed as such."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark designs share structural similarities or whether the models have seen similar Verilog patterns during training."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "RTLSquad improves Pass@1 by 7.2% over the Self-Planning baseline",
       "evidence": "Table II shows Pass@1 improvements: LLaMA 47.6→58.0 (+10.4pp), Deepseek 60.8→72.0 (+11.2pp). The specific 7.2% figure in the abstract cannot be derived from the reported data.",
       "supported": "weak"
     },
     {
       "claim": "RTLSquad matches or exceeds PPA of reference designs in most cases (73.3%)",
       "evidence": "Table I compares PPA metrics across 11 designs. RTLSquad shows improvements in several designs but fails entirely on 2 (right_shifter, width_8to16) and shows worse metrics on some (pulse_detect power/area). The exact 73.3% is hard to verify from the table.",
       "supported": "moderate"
     },
     {
       "claim": "The multi-agent system provides decision interpretability through inter-agent communication",
       "evidence": "Section IV provides three case study examples (implementation, verification, exploration) showing agent dialogue excerpts. No empirical evaluation of interpretability quality — no user study, no engineer feedback, no interpretability metric.",
       "supported": "weak"
     },
     {
       "claim": "Multi-round feedback and correction in the implementation stage provides better RTL code generation capabilities",
       "evidence": "Table II shows RTLSquad outperforms SP baseline on Pass@1. However, no ablation isolates the implementation stage's contribution from other components (exploration, verification stages).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Abstract claim does not match reported data",
       "detail": "The abstract states 'Pass@1 performance improving by 7.2%' but Table II shows improvements of 10.4pp (LLaMA) and 11.2pp (Deepseek), averaging 10.8pp. The 7.2% figure cannot be derived from the available results."
     },
     {
       "flag": "Core claim (interpretability) never empirically validated",
       "detail": "Decision interpretability is the paper's central contribution, presented as solving a key barrier to engineer trust and adoption. Yet no human evaluation, user study, or interpretability metric is used — only three cherry-picked case study excerpts demonstrate it."
     },
     {
       "flag": "No statistical testing despite comparative claims",
       "detail": "The paper makes comparative claims ('excels', 'demonstrates better capability') based on comparing point estimates with no significance tests, no error bars, and no multi-run results despite using temperature 0.8."
     },
     {
       "flag": "Compute asymmetry with baseline not addressed",
       "detail": "RTLSquad uses multiple iterative LLM calls across several agent squads per design while the SP baseline uses a single planning-then-generation call. The improvement may simply reflect more compute rather than better architecture."
     },
     {
       "flag": "Modified benchmark with no transparency",
       "detail": "The authors 'optimized the testbench files, adding more debugging outputs' — providing RTLSquad with enhanced feedback not available to the baseline. The specific modifications are not described, making the comparison potentially unfair."
     },
     {
       "flag": "Failures not analyzed",
       "detail": "RTLSquad completely failed on 2 of 11 designs (right_shifter, width_8to16) and produced worse results on pulse_detect, but these failures receive no discussion or analysis."
     }
   ],
   "cited_papers": [
     {
       "title": "RTLCoder: Fully Open-Source and Efficient LLM-Assisted RTL Code Generation Technique",
       "authors": ["S. Liu", "W. Fang", "Y. Lu", "J. Wang", "Q. Zhang", "H. Zhang", "Z. Xie"],
       "year": 2024,
       "relevance": "LLM-based RTL code generation via fine-tuning, directly relevant to AI code generation evaluation."
     },
     {
       "title": "Benchmarking large language models for automated verilog RTL code generation",
       "authors": ["S. Thakur", "B. Ahmad", "Z. Fan", "H. Pearce", "B. Tan", "R. Karri", "B. Dolan-Gavitt", "S. Garg"],
       "year": 2022,
       "relevance": "Benchmark evaluation of LLMs for Verilog code generation, directly relevant to LLM coding capabilities."
     },
     {
       "title": "VeriGen: A Large Language Model for Verilog Code Generation",
       "authors": ["S. Thakur", "B. Ahmad", "H. Pearce", "B. Tan", "B. Dolan-Gavitt", "R. Karri", "S. Garg"],
       "year": 2024,
       "relevance": "LLM fine-tuned for Verilog generation, relevant to AI code generation and domain-specific models."
     },
     {
       "title": "RTLRewriter: Methodologies for Large Models aided RTL Code Optimization",
       "authors": ["X. Yao", "Y. Wang", "X. Li", "Y. Lian", "R. Chen", "L. Chen", "M. Yuan", "H. Xu", "B. Yu"],
       "year": 2024,
       "relevance": "LLM-based RTL code optimization using multi-stage processes, directly comparable approach."
     },
     {
       "title": "Make Every Move Count: LLM-based High-Quality RTL Code Generation Using MCTS",
       "authors": ["M. DeLorenzo", "A. B. Chowdhury", "V. Gohil", "S. Thakur", "R. Karri", "S. Garg", "J. Rajendran"],
       "year": 2024,
       "relevance": "MCTS-guided LLM for RTL code generation, relevant to AI code generation and search-based methods."
     },
     {
       "title": "VerilogEval: Evaluating Large Language Models for Verilog Code Generation",
       "authors": ["M. Liu", "N. Pinckney", "B. Khailany", "H. Ren"],
       "year": 2023,
       "relevance": "Evaluation benchmark for LLM Verilog generation, relevant to benchmark design for AI coding."
     },
     {
       "title": "RTLFixer: Automatically Fixing RTL Syntax Errors with Large Language Models",
       "authors": ["Y.-D. Tsai", "M. Liu", "H. Ren"],
       "year": 2024,
       "relevance": "LLM-based automated RTL error correction using ReAct paradigm, relevant to agentic AI code repair."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": ["C. Qian", "W. Liu", "H. Liu", "N. Chen", "Y. Dang", "J. Li", "C. Yang", "W. Chen", "Y. Su", "X. Cong", "J. Xu", "D. Li", "Z. Liu", "M. Sun"],
       "year": 2024,
       "relevance": "Multi-agent LLM system for software development, directly relevant to agentic AI workflows."
     },
     {
       "title": "Self-planning Code Generation with Large Language Models",
       "authors": ["X. Jiang", "Y. Dong", "L. Wang", "Z. Fang", "Q. Shang", "G. Li", "Z. Jin", "W. Jiao"],
       "year": 2023,
       "relevance": "Planning-before-coding approach for LLM code generation, used as the baseline in this paper."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "relevance": "Introduces Pass@k evaluation metric for LLM code generation, foundational for AI coding evaluation."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["S. Yao", "J. Zhao", "D. Yu", "N. Du", "I. Shafran", "K. Narasimhan", "Y. Cao"],
       "year": 2023,
       "relevance": "Foundational agentic paradigm combining reasoning with tool use, relevant to LLM agent design."
     },
     {
       "title": "SynthAI: A Multi Agent Generative AI Framework for Automated Modular HLS Design Generation",
       "authors": ["S. A. Sheikholeslam", "A. Ivanov"],
       "year": 2024,
       "relevance": "Multi-agent LLM system for HLS design, directly comparable approach in hardware design domain."
     }
   ]
 }