ai-research-survey

scan.json (28810B)

---

 {
   "paper": {
     "title": "DRC-Coder: Automated DRC Checker Code Generation Using LLM Autonomous Agent",
     "authors": [
       "Chen-Chia Chang",
       "Chia-Tung Ho",
       "Yaguang Li",
       "Yiran Chen",
       "Haoxing Ren"
     ],
     "year": 2025,
     "venue": "ISPD '25 (International Symposium on Physical Design)",
     "arxiv_id": "2412.05311",
     "doi": "10.1145/3698364.3705347"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "DRC-Coder, a multi-agent framework combining GPT-4o with vision language models, achieves perfect F1=1.000 on all 7 design rules for automated DRC code generation in a sub-3nm technology node, compared to F1=0.631 for standard prompting. The system generates code per design rule in ~4 minutes on average (210 seconds, 2.3 debugging iterations). Ablation shows both multi-agent decomposition (F1=0.935 without vision) and vision capability (F1=0.911 single-agent) contribute, but the full system is needed for perfect scores.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL is provided. The paper does not mention releasing the DRC-Coder framework code, prompts, or utility functions."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The 207 standard cell layouts and DRC reports are proprietary (sub-3nm technology node, NVCell). No dataset download link is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Python language based on the multi-agent system development toolkit AutoGen' and 'GPT-4o using the OpenAI API version 2024-05-13' but provides no requirements.txt, library versions, or environment specification sufficient to recreate the setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The workflow is described conceptually but not with enough detail to reproduce."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only point estimates for Precision, Recall, and F1 with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims DRC-Coder outperforms standard prompting (37% higher F1) but provides no statistical significance tests. Comparisons are based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context: 'DRC-Coder achieves 37% higher F1 score' (from 0.631 to 1.000), and ablation variants show 32.5% and 30.7% improvements over standard prompting. Per-rule breakdowns in Table 1 provide baseline and method scores."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses 207 standard cell layouts and 7 design rules. No justification is given for why these numbers are sufficient, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be from single experimental runs with no indication of multiple trials."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Standard prompting with the same LLM (GPT-4o) is used as the primary baseline (Table 1). Two ablation variants are also compared (Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The authors are the first to address automated DRC code generation, so there are no prior methods. The standard prompting baseline with GPT-4o and the comparison with Llama3 are reasonable for a new problem domain."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 presents an ablation study with two variants: multi-agent without vision capability (avg F1=0.935) and single-agent with vision capability (avg F1=0.911), isolating the contribution of vision and multi-agent decomposition."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: Precision, Recall, and F1 score. The paper explains why F1 is the primary metric for imbalanced datasets (Section 4.3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated code. Evaluation is entirely automated by comparing generated code output against commercial DRC tool reports."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The 2 layout examples randomly selected for the prompt (Figure 6) are part of the same 207-layout dataset used for evaluation. No held-out test set is described, and no separation of prompt examples from evaluation data is mentioned."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-design-rule breakdown across all 7 rules with individual Precision, Recall, and F1 scores for each method."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The workflow case study (Figure 11) shows debugging iterations with specific false positives and false negatives. Figure 9 shows example false negative analysis. The Llama3 results demonstrate failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Llama3 results show the framework 'cannot perform as effective as GPT-4o' (avg F1=0.726). Ablation variants show imperfect performance. Standard prompting failures are detailed (e.g., M1.S.2 with Llama3 yields F1=0.000)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of F1=1.000 for DRC-Coder and F1=0.631 for standard prompting are supported by Table 1. The claim of 'four minutes on average' is supported by the 210-second average runtime in Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about multi-agent and vision capability contributions ('enhance the LLM reasoning ability'). The ablation study (Table 2) provides controlled single-variable manipulations that adequately support these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract specifies 'targeting on a sub-3nm technology node for a state-of-the-art standard cell layout tool.' The paper bounds evaluation to NVCell with 7 design rules. Future extensions are framed as 'potential' not claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. For example, the perfect F1 could be due to the simplicity of the 7 rules tested, the small evaluation set, or GPT-4o having seen similar DRC patterns in training data. None of these alternatives are considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures F1 score against commercial DRC tool reports and claims the generated code 'meets the standard of a commercial DRC tool.' The proxy (F1 against golden reports) closely matches the claimed outcome (correct DRC code). The paper explicitly discusses why F1 is appropriate for imbalanced DRC datasets."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 specifies 'GPT-4o using the OpenAI API version 2024-05-13.' Llama3 is referenced by name with citation [4], though no specific checkpoint is given. The GPT-4o API version provides adequate specificity."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figure 6 shows the initial prompt template with placeholders like '[DR Dependent Input]' that are dynamically changed. Figures 7-9 show tool function prompt templates. However, these are templates with varying inputs, and the Planner/Programmer agent system prompts (role definitions) are not provided. The reader cannot reconstruct every prompt sent to the model."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, max tokens, etc.) are reported for GPT-4o or Llama3. These significantly affect output and are not mentioned anywhere."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent scaffold is described in detail: Planner and Programmer roles (Section 4.2-4.3), group chat interaction, three tool functions (Foundry Rule Analysis, Layout DRV Analysis, DRC Code Evaluation), feedback loops, and iterative debugging. Figure 5 provides the overall architecture."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents data preparation: layout generation via NVCell with mutated routing behaviors (Section 3.1), and DRC report preprocessing converting physical coordinate-based reports to grid-based representation (Section 3.2). Figure 4 illustrates the conversion process."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. The conclusion mentions future extensions but does not discuss current limitations. No threats-to-validity section exists."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. No mention of potential issues with the small number of design rules, single technology node, or reliance on GPT-4o."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries stated. The paper does not enumerate what was NOT tested (e.g., more complex multi-layer rules, other technology nodes, other layout tools). The conclusion lists potential extensions but does not frame them as current limitations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. The 207 layouts, DRC reports, and generated code are not released. Results cannot be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes layout generation: '207 different standard cell layouts using NVCell by mutating the routing behaviors without DRC fixing.' Section 3.2 describes DRC report preprocessing with the commercial tool conversion to grid-based format."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data is computationally generated from a layout tool, not sampled from a population."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the full pipeline: layout generation via NVCell → commercial DRC tool reports → polygon-to-grid conversion → grid-based DRV ground truth. Figure 4 illustrates the conversion process."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgement section states: 'This work is supported in part by NVIDIA Corporation and NSF under Grant No. 2106828.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Chia-Tung Ho and Haoxing Ren at NVIDIA Research, Yaguang Li at NVIDIA, Chen-Chia Chang and Yiran Chen at Duke University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "NVIDIA funds the work and three authors are NVIDIA employees. The target system (NVCell) is NVIDIA's product. NVIDIA has a direct financial interest in demonstrating that LLMs can accelerate DRC code generation for their tools."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Three authors are NVIDIA employees working on NVIDIA's NVCell tool, but no formal conflict-of-interest declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of GPT-4o's or Llama3's training data cutoff date. The paper uses these models to generate code but does not discuss when their training data ends."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether design rule patterns or similar DRC code could appear in GPT-4o's training data. While the specific sub-3nm rules are proprietary, general DRC coding patterns may be in training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of contamination risk. The proprietary nature of sub-3nm design rules makes contamination less likely, but this is not acknowledged or analyzed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports runtime per design rule (45-354 seconds, average 210 seconds) and number of debugging iterations (1-3, average 2.3). The abstract states 'within four minutes on average.' However, no API cost in dollars or tokens consumed is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (total API spend, total tokens, or hardware used) is stated. Only per-rule runtime is provided."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or runs. LLM outputs are non-deterministic, but seed sensitivity is not addressed. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It appears results are from a single execution per configuration, but this is never explicitly confirmed."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is mentioned. The prompt design, number of examples, and agent configuration appear to be hand-designed without reporting a search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents the final DRC-Coder configuration without explaining how the prompt structure, tool function designs, or agent configurations were selected from alternatives."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement both their system and the standard prompting baseline. No acknowledgment of potential author-evaluation bias or discussion of how baseline implementation choices could affect the comparison."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "DRC-Coder uses iterative LLM calls (avg 2.3 iterations with tool function calls), consuming significantly more compute than single-pass standard prompting. This compute difference is not discussed or controlled for in the comparison."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses why F1 score is appropriate for imbalanced DRC datasets and explains that the metric measures alignment with the commercial DRC tool's reports. The paper connects its benchmark to the practical goal of replicating commercial tool results."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 2) isolates scaffold components: multi-agent vs single-agent, and with/without vision. The comparison between standard prompting (no scaffold) and DRC-Coder explicitly tests the scaffold as the variable of interest."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4o's training data could contain similar DRC coding patterns or design rule descriptions from publicly available EDA resources."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The prompt includes 2 layout examples with golden DRV locations from the same 207-layout evaluation set. This provides labeled examples from the test set as input, but this leakage is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 207 layouts are all generated from the same tool (NVCell) with similar structural characteristics. No discussion of whether this shared origin creates non-independence in the evaluation."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No analysis of whether training data overlap exists."
       }
     }
   },
   "claims": [
     {
       "claim": "DRC-Coder achieves perfect F1=1.000 on all 7 design rules using GPT-4o, compared to F1=0.631 for standard prompting.",
       "evidence": "Table 1 shows per-rule Precision, Recall, and F1 for all methods. DRC-Coder with GPT-4o achieves 1.000 across all metrics for all 7 rules. Standard prompting with GPT-4o averages P=0.690, R=0.624, F=0.631.",
       "supported": "moderate"
     },
     {
       "claim": "DRC-Coder generates code per design rule within four minutes on average (210 seconds, 2.3 debugging iterations).",
       "evidence": "Table 1 reports runtime per rule: 45-354 seconds with average 210 seconds and average 2.3 iterations.",
       "supported": "moderate"
     },
     {
       "claim": "Both multi-agent decomposition and vision capability contribute to DRC-Coder's performance.",
       "evidence": "Table 2 ablation: multi-agent without vision achieves avg F1=0.935; single-agent with vision achieves avg F1=0.911. Full system achieves 1.000. Both variants outperform standard prompting (F1=0.631) but fall short of the complete system.",
       "supported": "moderate"
     },
     {
       "claim": "DRC-Coder with Llama3 also improves over standard prompting (72.6% vs 42.1% F1).",
       "evidence": "Table 1 shows Llama3 standard prompting averages F1=0.421 while DRC-Coder with Llama3 averages F1=0.726, a 42.2% improvement.",
       "supported": "moderate"
     },
     {
       "claim": "DRC-Coder significantly reduces development time from days of manual effort to minutes.",
       "evidence": "Section 6 states DRC-Coder 'drastically reduces the coding time from days of manual effort to an average of four minutes per design rule.' The 'days' estimate for manual work is not substantiated with data.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Perfect scores with no variance",
       "detail": "DRC-Coder achieves F1=1.000 on all 7 rules with no variance reported. LLM outputs are non-deterministic, yet there is no indication of multiple runs or any result variation. Perfect scores on every single rule, with no single false positive or false negative across 207 layouts, warrants skepticism without repeated trials."
     },
     {
       "flag": "NVIDIA conflict of interest",
       "detail": "Three of five authors are NVIDIA employees. The work is funded by NVIDIA. The target system (NVCell) is NVIDIA's product. The paper demonstrates that LLMs can accelerate development of NVIDIA's proprietary tools. No conflict-of-interest statement is provided."
     },
     {
       "flag": "Narrow evaluation scope",
       "detail": "Only 7 design rules are tested, all for one technology node with one layout tool. Sub-3nm nodes can have hundreds of design rules. The paper title ('Automated DRC Checker Code Generation') implies broader applicability than what is demonstrated."
     },
     {
       "flag": "Test data leakage in prompts",
       "detail": "Two labeled examples from the 207-layout evaluation set are included in each prompt. The model receives ground-truth DRV locations for test examples during generation. The DRC Code Evaluation function also runs on the full 207 layouts during the iterative debugging loop, giving the agent feedback on the test set."
     },
     {
       "flag": "No reproducibility",
       "detail": "No code, no data, no prompts, no hyperparameters. The evaluation uses proprietary sub-3nm design rules and commercial DRC tool reports that cannot be independently verified."
     },
     {
       "flag": "Unsubstantiated manual effort baseline",
       "detail": "The claim of reducing 'days of manual effort' to minutes is not supported by measured data on manual coding time. The comparison to human engineers is anecdotal."
     }
   ],
   "cited_papers": [
     {
       "title": "Autogen: Enabling next-gen llm applications via multi-agent conversation framework",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Multi-agent LLM framework used as the development platform for DRC-Coder."
     },
     {
       "title": "Swe-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E Jimenez", "Alexander Wettig"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "LLM agent for automated software engineering with auto-debugging, directly related to agentic coding."
     },
     {
       "title": "React: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational LLM agent framework combining reasoning and acting, cited as core agent methodology."
     },
     {
       "title": "The llama 3 herd of models",
       "authors": ["Abhimanyu Dubey", "Abhinav Jauhri"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Open-source LLM used as alternative to GPT-4o in the evaluation, relevant to LLM capability comparison."
     },
     {
       "title": "Phi-3 technical report: A highly capable language model locally on your phone",
       "authors": ["Marah Abdin"],
       "year": 2024,
       "arxiv_id": "2404.14219",
       "relevance": "Small vision-language model compared against GPT-4o for design rule image interpretation."
     },
     {
       "title": "Codet: Code generation with generated tests",
       "authors": ["Bei Chen", "Fengji Zhang", "Anh Nguyen"],
       "year": 2022,
       "arxiv_id": "2207.10397",
       "relevance": "LLM code generation with automated testing, related to auto-debugging approach used in DRC-Coder."
     },
     {
       "title": "VerilogCoder: Autonomous Verilog Coding Agents with Graph-based Planning and Abstract Syntax Tree (AST)-based Waveform Tracing Tool",
       "authors": ["Chia-Tung Ho", "Haoxing Ren", "Brucek Khailany"],
       "year": 2024,
       "arxiv_id": "2408.08927",
       "relevance": "LLM agent for hardware description language code generation, closely related domain-specific coding agent."
     },
     {
       "title": "A survey on large language model based autonomous agents",
       "authors": ["Lei Wang", "Chen Ma", "Xueyang Feng"],
       "year": 2024,
       "relevance": "Survey of LLM-based autonomous agents, providing context for the agent framework methodology."
     },
     {
       "title": "Codegen2: Lessons for training llms on programming and natural languages",
       "authors": ["Erik Nijkamp", "Hiroaki Hayashi"],
       "year": 2023,
       "arxiv_id": "2305.02309",
       "relevance": "LLM training for code generation, relevant to understanding code generation model capabilities."
     },
     {
       "title": "Webshop: Towards scalable real-world web interaction with grounded language agents",
       "authors": ["Shunyu Yao", "Howard Chen", "John Yang"],
       "year": 2022,
       "relevance": "LLM agent for web interaction tasks, cited as example of agent capability in real-world domains."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "DRC engineers at semiconductor companies could potentially use this approach, but it requires proprietary tools and data, limiting immediate adoption."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Multi-agent LLM for domain-specific code generation follows expected trends; not contrarian, though applying it to DRC is novel."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; this is a specialized engineering automation tool."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle; straightforward engineering contribution."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released. Requires proprietary NVIDIA tools and sub-3nm design rules to reproduce."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "NVIDIA involvement adds some brand recognition, but DRC code generation is a niche EDA topic with limited mainstream visibility."
     }
   }
 }