ai-research-survey

scan-v5.json (26710B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "LAAFD: LLM-based Agents for Accelerated FPGA Design",
     "authors": [
       "Maxim Moraru",
       "Kamalavasan Kamalakkannan",
       "Jered Dominguez-Trujillo",
       "Patrick Diehl",
       "Atanu Barai",
       "Julien Loiseau",
       "Zachary Kent Baker",
       "Howard Pritchard",
       "Galen M. Shipman"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.06085",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims (99.9% performance, SODA parity, readable kernels) are supported by Section V results in Tables I and IV with corresponding cycle count and complexity data.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section V.E states 'only one or two runs produced optimal designs' out of 10 per kernel, yet abstract claims 99.9% performance. Reporting best-of-run results rather than typical performance undermines the causal claim that the workflow reliably produces this quality.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Tested on 15 kernels on one FPGA (xcu250) with one HLS tool (Vitis 2022.2), yet abstract claims the approach 'substantially lowers the expertise barrier to FPGA acceleration' and conclusion states it 'has the potential to broaden adoption.' Scope far exceeds tested domain.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Paper does not discuss whether models are memorizing from training data, whether simpler heuristics could achieve similar results, or whether optimization feedback is actually necessary versus random iteration.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Measured outcomes (execution cycles, lines of code, cyclomatic complexity) directly match claimed outcomes (latency performance, code readability) with no conflation.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated 'Limitations' or 'Threats to Validity' section. Issues are scattered in Section V.E discussion and conclusion rather than systematically presented.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section V.E mentions stochasticity and context limits buried in discussion, but lacks systematic threat analysis. The critical fact that 'only one or two runs produced optimal designs' is treated as an evaluation note, not a validity threat.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Paper specifies 15 HPC kernels and one FPGA device tested, but does not explicitly state non-applicability to: other HLS tools, embedded FPGAs, kernels exceeding context limits, or production workloads.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgment discloses Los Alamos National Laboratory, Triad National Security, and Department of Energy funding under contract 89233218CNA000001.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All nine authors listed with Los Alamos National Laboratory affiliation in header.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funder (US Department of Energy) is independent from the outcome (LAAFD tool). Not evaluating a vendor's product for the vendor.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, no declaration of patents, equity interests, or consulting relationships related to FPGA tools or synthesis frameworks.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": false,
         "justification": "Key terms like 'agent', 'optimization', 'synthesis', 'latency' are used without upfront definitions. Section II provides HLS background but assumes FPGA and hardware design familiarity.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Contributions explicitly stated in Section I: (1) LAAFD workflow, (2) comparison of LLM models, (3) performance evaluation. Intended contribution is unambiguous.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section III engages with 20+ prior works (Verigen, RTLCoder, HLSPilot, C2HLSC), explains evolution from one-shot to iterative approaches, and claims 'to the best of our knowledge, this is the first tool capable of reaching minimum theoretical latency across a wide range of kernels.'",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "Paper describes LAAFD system and methodology but does not mention releasing the tool, agents, or test code. No repository, GitHub link, or artifact descriptor provided.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "The 15 kernel descriptions are provided in Table I and Figures 3-8, but source code is not released. SODA kernels referenced but not provided. Generated HLS outputs not published.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": true,
           "justification": "Table III specifies Vitis 2022.2, FPGA device xcu250-figd2104-2L-e, 200 MHz frequency, 128-bit vector path, and model names (GPT-5, GPT-5-nano, o4-mini). Environment is reproducible.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Paper describes the workflow architecture and example (Figures 3-8) but provides no step-by-step instructions for setting up the system, running agents, or interpreting results.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Tables I and IV report single point estimates (ideal, manual, LAAFD). Figure 9 shows model comparison but no error bars, confidence intervals, or variance estimates.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "Comparisons are descriptive (99.9% geomean, matches SODA) with no statistical significance tests, p-values, or hypothesis tests reported.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Effect sizes reported as percentages of baseline (99.9%, 99.5%, 52.5%, 32.7%), code complexity ratios (0.89×, 2.27×), and success counts (6/15, 10/15, 15/15 kernels above 98%).",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "15 kernels are described as 'representing common compute patterns' but no justification for why 15 specifically. No power analysis or sample size calculation provided.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Section V.E states 'we report for each kernel the best HLS design obtained across multiple executions' and notes high run-to-run variance (only 1-2/10 optimal), but no distribution, mean, std dev, or min/max reported.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Hand-tuned baselines provided for all 15 kernels (Table I). SODA framework results provided for 7 stencil kernels (Table IV).",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": false,
           "justification": "SODA baseline is from 2018 (8 years old). Recent tools mentioned in related work (RTLCoder 2025, VerilogCoder 2025, AutoChip, RTLFixer) are not evaluated. Baselines are dated.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": false,
           "justification": "Section V.D compares three LLM models but no ablation of workflow components (translator, validator, compile fixer, runtime fixer, judge, optimizer). No study of iteration necessity or feedback importance.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Metrics include execution cycles (latency), FPGA resource usage (LUTs, FFs, DSPs, BRAMs), lines of code, and cyclomatic complexity. Multiple perspectives reported.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable. System automatically generates and validates code; no subjective human judgment of output quality needed.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": false,
           "justification": "Same 15 kernels used for optimization feedback and final evaluation. Validation uses reference test on the same kernels, not held out. No train/test split.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results broken down by kernel (15 different compute patterns in Table I), by benchmark source (7 SODA kernels in Table IV), and by model (Figure 9 shows GPT-5, GPT-5-nano, o4-mini).",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Section V.E briefly mentions 'only one or two runs produced optimal designs' and context size limits for larger kernels, but no systematic analysis or specific failure examples shown.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": false,
           "justification": "Section V.D shows GPT-5-nano and o4-mini underperform on complex kernels (32.7% and 52.5% geomean), and context limitations mentioned, but framed as model comparison rather than negative results.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Model names given (gpt-5, gpt-5-nano, o4-mini) but no exact version IDs, snapshot dates, or parameter counts specified. Cannot reproduce with specific model versions.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "Section IV describes what agents do conceptually ('translator receives C++ kernel...task is to translate to HLS') but actual system prompts, instructions, or templates are not provided.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "Temperature, top-p, sampling strategy not specified. Iteration limits mentioned as 'predefined' and 'fixed' but values not given (Section V.E mentions 25 for SODA kernels but not for others).",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "Agentic workflow extensively described in Section IV.A with roles for translator, validator, compile fixer, runtime fixer, judge, and optimizer. Figure 2 and detailed example (Figures 3-8) clarify the pipeline.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable. Kernels are direct C++ inputs with reference tests; no preprocessing pipeline for data collection or feature engineering.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Kernel specifications shown in Table I and example code (Figures 3-8), but source code, test benches, and reference implementations not released for independent verification.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": false,
           "justification": "'15 kernels representing common compute patterns in HPC' stated but selection criteria not described. SODA kernels taken from [1] but rationale for this set not explained.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable. No human subjects or data collection from recruitment.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable in traditional sense. The workflow pipeline (translator → validator → optimizer) is documented, but no data collection or ETL pipeline.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Model training cutoffs not stated for GPT-5, GPT-5-nano, or o4-mini. These are closed-source models with unknown training data cutoff dates.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Paper acknowledges 'relatively small amount of FPGA-related source code available on GitHub, which likely influenced LLM training' but does not assess whether these specific 15 kernels could have been in training data.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Kernels are not standard public benchmarks (custom 15 kernels + SODA set), reducing contamination risk. However, no explicit discussion of whether specific optimization patterns might exist in training data.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. No human participants or user studies.",
         "source": "haiku"
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Conclusion states 'cost of approximately US$50 to translate and optimize all 15 kernels highlights the feasibility' with caveat that 'server infrastructure costs are not included.'",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Only aggregate cost (~$50) reported. No per-kernel costs, iteration costs, or breakdown by agent. Infrastructure costs explicitly excluded.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "LAAFD achieves 99.9% geomean performance compared to hand-tuned baselines across 15 kernels",
       "evidence": "Table I reports cycle counts for all 15 kernels; geometric mean calculated as 99.9% of manual baseline",
       "supported": "moderate"
     },
     {
       "claim": "For stencil workloads, LAAFD matches SODA performance while producing more readable code",
       "evidence": "Table IV shows comparable cycle counts; Table IV shows 8.3× fewer lines of code and 2.27× lower cyclomatic complexity than SODA",
       "supported": "strong"
     },
     {
       "claim": "GPT-5 outperforms GPT-5-nano and o4-mini, achieving 99.9% on all 15 kernels",
       "evidence": "Section V.D and Figure 9 show per-model performance; GPT-5 achieves ≥98% on all 15, others fail on 5-9 kernels",
       "supported": "strong"
     },
     {
       "claim": "The workflow generates functionally correct HLS kernels through iterative translation, validation, and optimization",
       "evidence": "Section IV describes workflow phases; Section IV.B illustrates 7-iteration example where code converges from 65,543 cycles to 16,396 cycles",
       "supported": "strong"
     },
     {
       "claim": "Results depend heavily on model quality; only 1-2 of 10 runs achieve theoretical minimum for complex kernels",
       "evidence": "Section V.E states 'for more complex SODA kernels, we ran 10 times...only one or two runs produced optimal designs'",
       "supported": "strong"
     },
     {
       "claim": "Generated LAAFD kernels consume more FPGA resources than hand-tuned or SODA baselines",
       "evidence": "Tables IV and V show up to 5.7× higher BRAM usage (S3D: 584 vs 122) and generally higher LUT/FF counts",
       "supported": "strong"
     },
     {
       "claim": "LLM-based agentic workflows substantially lower the expertise barrier to FPGA acceleration",
       "evidence": "Conceptual claim in abstract and conclusion; no empirical evidence (no user study, no comparison of developer productivity)",
       "supported": "unsupported"
     },
     {
       "claim": "LAAFD solves context size limitations by removing HLS report details or eliminating the judge session",
       "evidence": "Section V.E mentions 'we addressed this by summarizing HLS reports or...removing the dedicated judge session' but scope and impact not quantified",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study",
     "empirical"
   ],
   "key_findings": "LAAFD uses a multi-agent LLM-based workflow (translator, validator, fixer components, judge, optimizer) to convert C++ kernels to optimized Vitis HLS code, achieving 99.9% geomean performance versus hand-tuned baselines on 15 HPC kernels and matching SODA on stencil workloads. The workflow is inherently stochastic, with only 1-2 out of 10 runs producing theoretically optimal designs for complex kernels; GPT-5 substantially outperforms GPT-5-nano (99.9% vs 32.7%) and o4-mini (99.9% vs 52.5%). While LAAFD produces more readable code than SODA (8.3× fewer lines, 2.27× lower complexity), generated kernels consume significantly more FPGA resources (up to 5.7× more BRAMs in S3D), and the workflow encounters context size limitations on large kernels.",
   "red_flags": [
     {
       "flag": "Best-of-run cherry-picking",
       "detail": "Section V.E reports 'the best HLS design obtained across multiple executions' but notes that 'only one or two runs produced optimal designs' out of 10. Reporting 99.9% performance obscures that most runs were suboptimal. No distribution of run outcomes provided."
     },
     {
       "flag": "No variance reported",
       "detail": "High run-to-run stochasticity acknowledged (10 runs, 25 iterations per complex kernel) but results shown only as best-case. Mean, median, std dev, min/max performance not provided, making typical performance unknown."
     },
     {
       "flag": "Small, unjustified sample size",
       "detail": "15 kernels chosen to 'represent common compute patterns' but no justification for why 15 vs 10 vs 50. No power analysis or sample size calculation."
     },
     {
       "flag": "Dated baseline comparisons",
       "detail": "SODA baseline from 2018 (8 years old). Recent tools mentioned in related work (RTLCoder 2025, VerilogCoder 2025, AutoChip 2024) not evaluated, preventing assessment vs state-of-the-art."
     },
     {
       "flag": "Model versions not specified",
       "detail": "Model names given (GPT-5, GPT-5-nano, o4-mini) but no snapshot dates, version IDs, or parameter counts. Cannot reproduce with the exact same models."
     },
     {
       "flag": "System prompts not released",
       "detail": "Paper describes what agents do conceptually but provides no actual system instructions, prompt templates, or few-shot examples. Cannot reproduce the exact guidance given to models."
     },
     {
       "flag": "Hyperparameters underspecified",
       "detail": "Temperature, top-p, iteration limits stated as 'predefined' and 'fixed' (25 iterations mentioned for SODA kernels in V.E but not for others), validation thresholds not quantified."
     },
     {
       "flag": "Resource overhead unexplored",
       "detail": "Section V.B notes 'workflow was not instrumented to optimize resource utilization' and that LAAFD 'generally incurs higher resource usage.' BRAM usage up to 5.7× baseline suggests potential productionization barrier not addressed."
     },
     {
       "flag": "Context size limitations mentioned but not solved",
       "detail": "Section V.E notes 'executions sometimes exceeded maximum context size' for larger kernels; remedy (summarizing reports, removing judge) degrades quality but scope/frequency of this issue not quantified."
     },
     {
       "flag": "No ablation study",
       "detail": "Cannot determine which workflow components are necessary. No study of feedback importance vs random iteration, or necessity of compile fixer vs runtime fixer."
     },
     {
       "flag": "Training data contamination not addressed",
       "detail": "Paper acknowledges limited FPGA code in training but does not assess whether these specific 15 kernels or the optimization patterns might have been in GPT-5 training data."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Comparisons reported as descriptive percentages (99.9%, 52.5%) with no p-values, confidence intervals, or significance tests."
     }
   ],
   "cited_papers": [
     {
       "title": "SODA: Stencil with Optimized Dataflow Architecture",
       "relevance": "State-of-the-art stencil HLS generator; primary baseline for comparison on 7 stencil kernels (Table IV)"
     },
     {
       "title": "StencilFlow: Mapping large stencil programs to distributed spatial computing systems",
       "relevance": "Prior work on stencil compilation; referenced for domain-specific HLS approaches"
     },
     {
       "title": "HLS4ML: An open-source codesign workflow to empower scientific low-power machine learning devices",
       "relevance": "High-level synthesis framework for ML; demonstrates application of HLS to specialized domain"
     },
     {
       "title": "VeriGen: A Large Language Model for Verilog Code Generation",
       "relevance": "RTL generation using LLMs; foundational work on LLM-based hardware synthesis (Thakur et al. 2024)"
     },
     {
       "title": "RTLCoder: Fully Open-Source and Efficient LLM-Assisted RTL Code Generation Technique",
       "relevance": "Iterative RTL synthesis with EDA feedback; similar agentic approach applied to Verilog (Liu et al. 2025)"
     },
     {
       "title": "C2HLSC: Leveraging Large Language Models to Bridge the Software-to-Hardware Design Gap",
       "relevance": "LLM-based C-to-HLS translation with design space exploration; prior work on same problem (Collini et al. 2025)"
     },
     {
       "title": "A Survey on Code Generation with LLM-based Agents",
       "relevance": "Recent survey of agentic code generation methods; contextualizes LAAFD within broader LLM agent ecosystem (Dong et al. 2025)"
     },
     {
       "title": "AutoChip: Automating HDL Generation Using LLM Feedback",
       "relevance": "Multi-turn HDL generation with LLM-as-judge pattern; precursor to judge-based optimization strategy"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Tool could help HLS developers, but is not released; resource overhead (5.7× BRAMs in some cases) raises productionization concerns for resource-constrained FPGAs."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "LLMs for code generation and optimization well-established; applying them to HLS is somewhat novel but unsurprising given trends in hardware-software co-design."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety concerns; paper focuses on productivity and code generation, not autonomy or alignment."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Tool not released; no code repository, no online demo, no downloadable artifact. Readers cannot try LAAFD themselves."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Straightforward engineering paper with no controversial claims, no disputes with prior work, no shocking findings."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Los Alamos National Laboratory carries credibility in HPC, but not as high-profile as OpenAI/DeepMind/Google for LLM work."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "46893430",
         "title": "The Trigger in the Haystack: Extracting and Reconstructing LLM Backdoor Triggers",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46893430",
         "created_at": "2026-02-04T23:28:00Z"
       }
     ],
     "top_points": 1,
     "total_points": 1,
     "total_comments": 0
   }
 }