ai-research-survey

scan.json (18209B)

---

 {
   "paper": {
     "title": "CreativEval: Evaluating Creativity of LLM-Based Hardware Code Generation",
     "authors": ["Matthew DeLorenzo", "Vasudev Gohil", "Jeyavijayan Rajendran"],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2404.08806"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided: https://github.com/matthewdelorenzo/CreativEval/ stated in the introduction as open-source."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states datasets will be open-sourced at the same GitHub link. The prompts are sourced from publicly available HDLBits via AutoChip."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Python 3.10, Icarus Verilog 10.3, and NVIDIA A100 GPU, but does not provide a requirements.txt, Dockerfile, or detailed dependency list sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table I reports only point estimates for all metrics. No confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GPT-3.5 is the most creative and makes comparative claims between models, but no significance tests are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw metric values are reported. No effect sizes with baseline context (e.g., Cohen's d or percentage improvement with denominators) are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset consists of 111 single-module prompts and 9 multi-module prompts. No justification is given for why these sizes are sufficient, and the elaboration set (9 prompts) is notably small."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across the t=10 responses per prompt. Only aggregated metrics are shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple LLMs are compared against each other (CodeLlama-7B/13B, VeriGen-6B/16B, GPT-3.5, GPT-4), serving as mutual baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models evaluated (GPT-4, GPT-3.5, CodeLlama, VeriGen) were contemporary at the time of writing (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed to evaluate which components of the creativity framework matter most, e.g., varying the equal weighting or the GNN4IP threshold."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four creativity sub-metrics (fluency, flexibility, originality, elaboration) plus overall creativity and functionality are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Creativity is inherently subjective, yet no human evaluation of the generated code's creativity is included. All evaluation is automated via GNN4IP similarity scores."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a training/tuning study. The framework evaluates pre-trained models on a fixed prompt set, so held-out test sets are not applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table I provides per-subcategory breakdowns (fluency, flexibility, originality, elaboration) for each model."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure modes: CodeLlama producing direct copies for flexibility, VeriGen generating repeated similar implementations for fluency."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "GPT models performing worse on originality than smaller models is reported, and CodeLlama-7B scoring 0.0 on flexibility is noted."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims GPT-3.5 is the most creative model, which is supported by Table I showing GPT-3.5 with the highest creativity score (0.2201)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper suggests causal explanations (e.g., GPT models' lower originality 'could be due to its large size and training dataset') without adequate evidence. These are speculative."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to evaluate 'creativity of LLM-based hardware code generation' broadly, but results are limited to 111 HDLBits prompts in Verilog with a specific GNN4IP-based metric. This narrow scope is not bounded in the claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. For example, the originality metric may reflect training data memorization rather than creativity, and the GNN4IP similarity threshold choice is not examined."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-3.5' and 'GPT-4' without specifying API versions or snapshot dates. CodeLlama and VeriGen are specified by parameter count and HuggingFace links."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt examples are provided in Listings 1, 2, and 3 for fluency/originality, flexibility, and elaboration respectively."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A specifies temperature=0.3, max_tokens=1024, top_k=10, top_p=0.95 for all models."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The framework is direct prompt-response evaluation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A describes that prompts are sourced from HDLBits via AutoChip, responses are trimmed to the first 'endmodule', and functionality is checked via Icarus Verilog testbenches."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do not show. The conclusion only mentions future work evaluating more LLMs and larger prompt sets."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (individual GNN4IP similarity scores, per-prompt results) is provided in the paper. Only aggregated metrics in Table I."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A describes the prompt dataset (111 HDLBits prompts via AutoChip, 9 multi-module prompts), the LLM inference process, and the GNN4IP comparison procedure."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is a standard benchmark (HDLBits)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from prompt generation to LLM response to functionality check to GNN4IP similarity scoring to metric aggregation is documented in Section III and IV."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgment section lists Purdue CSME and NSF grants (CNS-1822848, DGE-2039610)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are from Texas A&M University. They evaluate third-party models (GPT, CodeLlama, VeriGen), no direct product conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF and Purdue CSME are independent of the LLM vendors evaluated. No funder has a stake in the results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training cutoff dates are stated for any of the models evaluated. HDLBits prompts may be in the training data of GPT models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HDLBits prompts or solutions appeared in the training data of the evaluated models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HDLBits is publicly available and could be in the training data of GPT-3.5/4. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API spend, or wall-clock time is reported despite querying GPT-3.5/4 APIs and running local inference on A100 GPU."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated. The hardware (A100 80GB) is mentioned but total GPU hours or API costs are not."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-3.5 is the most creative model for hardware code generation among those evaluated.",
       "evidence": "Table I shows GPT-3.5 achieves the highest overall creativity score of 0.2201, driven primarily by the highest flexibility score (0.1600).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 has the highest fluency in generating unique Verilog solutions.",
       "evidence": "Table I shows GPT-4 fluency of 0.1644, the highest among all models.",
       "supported": "moderate"
     },
     {
       "claim": "Creativity slightly drops for larger model sizes of GPT and VeriGen.",
       "evidence": "Table I: GPT-3.5 (0.2201) > GPT-4 (0.2107); VeriGen-6B (0.2026) > VeriGen-16B (0.1962). But differences are small and no statistical tests are performed.",
       "supported": "weak"
     },
     {
       "claim": "CodeLlama models produce direct copies when asked for alternative implementations, indicating flexibility depends on natural language understanding.",
       "evidence": "Section IV-B states CodeLlama 'produced results that were often direct copies of the provided module' with CodeLlama-7B scoring 0.0 on flexibility.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CreativEval proposes a framework measuring four creativity dimensions (fluency, flexibility, originality, elaboration) for LLM-based Verilog code generation using GNN4IP circuit similarity. Across six models, GPT-3.5 scored highest overall (0.2201) primarily due to superior flexibility, while GPT-4 led in fluency. Larger model variants showed slightly lower creativity scores, though differences are small and not statistically tested. All creativity scores are low in absolute terms (max 0.22 out of 1.0).",
   "red_flags": [
     {
       "flag": "No statistical rigor",
       "detail": "All comparative claims between models are based on single point estimates with no confidence intervals, significance tests, or variance reporting across the t=10 responses."
     },
     {
       "flag": "Benchmark contamination risk",
       "detail": "HDLBits prompts are publicly available and likely in GPT-3.5/4 training data. This could confound creativity metrics, especially originality, which measures deviation from 'golden' solutions the model may have memorized."
     },
     {
       "flag": "No human validation of creativity metric",
       "detail": "The paper proposes to measure 'creativity' using an automated graph similarity metric (GNN4IP) but never validates whether this metric correlates with human judgments of creativity in hardware design."
     },
     {
       "flag": "Very small elaboration dataset",
       "detail": "The elaboration metric is measured on only 9 multi-module prompts, producing nearly identical scores for 5 of 6 models (0.3333), making the metric uninformative."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no discussion of limitations, threats to validity, or scope boundaries."
     },
     {
       "flag": "Equal weighting assumption unjustified",
       "detail": "The four creativity sub-components are weighted equally (0.25 each) in the composite score with no justification or sensitivity analysis for this choice."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "relevance": "Foundational HumanEval benchmark paper for evaluating LLM code generation capabilities."
     },
     {
       "title": "VeriGen: A large language model for verilog code generation",
       "authors": ["S. Thakur", "B. Ahmad", "H. Pearce"],
       "year": 2023,
       "relevance": "Demonstrates fine-tuning smaller models for hardware RTL code generation, directly evaluated in this paper."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2024,
       "relevance": "Technical report for GPT-4, one of the primary models evaluated for creativity."
     },
     {
       "title": "ChipNeMo: Domain-adapted LLMs for chip design",
       "authors": ["M. Liu", "T.-D. Ene", "R. Kirby"],
       "year": 2024,
       "relevance": "Domain adaptation of LLMs for hardware design tasks via fine-tuning."
     },
     {
       "title": "RTLCoder: Outperforming GPT-3.5 in design RTL generation with our open-source dataset and lightweight solution",
       "authors": ["S. Liu", "W. Fang", "Y. Lu"],
       "year": 2024,
       "relevance": "Open-source RTL generation model that outperforms GPT-3.5, relevant to LLM code generation benchmarking."
     },
     {
       "title": "AutoChip: Automating HDL generation using LLM feedback",
       "authors": ["S. Thakur", "J. Blocklove", "H. Pearce"],
       "year": 2023,
       "relevance": "LLM-based automated hardware design with feedback loops, source of the prompt dataset used in this paper."
     },
     {
       "title": "Assessing and understanding creativity in large language models",
       "authors": ["Y. Zhao", "R. Zhang", "W. Li"],
       "year": 2024,
       "relevance": "Evaluates LLM creativity using the same four cognitive sub-categories but for natural language tasks."
     },
     {
       "title": "VerilogEval: Evaluating large language models for Verilog code generation",
       "authors": ["M. Liu", "N. Pinckney", "B. Khailany"],
       "year": 2023,
       "relevance": "Benchmark for evaluating LLM functionality in Verilog generation."
     }
   ]
 }