ai-research-survey

scan.json (20692B)

---

 {
   "paper": {
     "title": "CudaForge: An Agent Framework with Hardware Feedback for CUDA Kernel Optimization",
     "authors": ["Zijian Zhang", "Rong Wang", "Shiyang Li", "Yuebo Luo", "Mingyi Hong", "Caiwen Ding"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2511.01884"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided in abstract: https://github.com/OptimAI-Lab/CudaForge"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "They use the publicly available KernelBench benchmark (250 tasks, Levels 1-3). The benchmark is publicly accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup with library versions is described in the paper. GPUs are mentioned but software dependencies are not specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but the paper itself does not contain a 'Reproducing Results' section or specific commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., 97.6% correctness, 1.677x speedup) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CudaForge outperforms baselines based solely on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout, e.g., '1.677x speedup over PyTorch baselines' and comparisons like Kevin-32B's 1.10x vs CudaForge's 1.662x. Percentage improvements and absolute speedup values are consistently given."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The stratified subset D* uses only 25 tasks but no justification is given for why 25 tasks is sufficient. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across runs. Results appear to be from single runs per configuration."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: OpenAI-o3 (one-shot), o3-self-refine, o3-correction, o3-optimization, Kevin-32B, Agentic Baseline, and CudaForge(full metrics)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Kevin-32B (2025), Agentic Baseline (2025), and OpenAI-o3 (2025), all contemporary methods."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive ablation studies in Section 3.6: o3-self-refine (no Judge), o3-correction (correction-only), o3-optimization (optimization-only), full metrics vs. subset metrics. Section 3.7 covers GPU and base model variations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five metrics are used: Correctness, Performance (average speedup), Fast1, Median speedup, and 75th percentile speedup."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated kernels' quality beyond automated correctness checking. The paper mentions manually checking for 'fake kernels' in CUDA-L1 but this is not a systematic human evaluation of CudaForge outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The NCU metric selection procedure uses representative tasks (Conv2D, MatMul) from KernelBench, then the full system is evaluated on KernelBench. There is no discussion of whether metric selection tasks overlap with evaluation tasks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-level breakdowns (Level 1, 2, 3) across all metrics. Table 5 provides per-model breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 (Case study) shows both optimization and correction (failure/repair) rounds. The case study in Figure 8 includes a correction round where a kernel failed numerical checks."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that using full NCU metrics leads to worse performance than the subset (Table 1), and that some model combinations (e.g., QwQ/O3) achieve only 0.790x average speedup (below 1x). Claude-Sonnet-4 as Coder achieves only 88% correctness."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 97.6% correctness, 1.68x speedup, 2.27x scaling up, generalization across GPUs and base models, and $0.3 cost are all supported by Tables 1-5 and Section 3.5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about the importance of hardware feedback, Judge role separation, and metric selection are supported by controlled ablation studies (Section 3.6) that isolate individual components."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim 'CUDA Kernel Optimization' broadly, but results are limited to KernelBench tasks (250 tasks of specific types). The paper does not explicitly bound claims to this benchmark setting. Claims like 'cost-effective, generalizable, and high-performance CUDA kernel optimization' extend beyond what was tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the results. For example, the performance gains could partly be due to the choice of base model (o3) rather than the framework design. No threats-to-validity or confound analysis is presented."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'OpenAI-o3', 'GPT-5', 'Claude-Sonnet-4', 'GPT-OSS-120B', 'QwQ-32B' without specifying API versions or snapshot dates. These are marketing names without version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts for the Coder (initial round, correction, optimization) and Judge (correction, optimization) are provided in Appendix A with actual text, including output format specifications and rules."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other LLM API sampling parameters are reported. Only N=10 maximum iteration rounds and tolerance 1e-4 are stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail: two-agent Coder/Judge architecture, iterative workflow, lightweight memory design (no conversation history), correction vs. optimization modes, NCU profiling integration, and JSON-structured feedback (Section 2)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The NCU metric selection pipeline is documented in detail (Algorithms 1-2): kernel sampling, Top-20 per-task selection via Pearson correlation, cross-task consolidation with P75 threshold. The stratified subset D* construction is described in Section 3.1 and Appendix D.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations or Threats to Validity section in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or what settings are excluded. No explicit boundary statements about generalization limits."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-task speedups, generated kernels, NCU profiles) is released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: KernelBench tasks (Levels 1-3, 250 tasks), stratified random sampling for D* (25 tasks), NCU profiling procedure, and correctness test criteria (Section 3.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; the study uses a standard public benchmark (KernelBench)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from task input to final kernel selection is documented: Coder generates kernel, compilation test, execution test with 1e-4 tolerance, NCU profiling, Judge feedback, iterative refinement, best correct kernel selection (Section 2)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with University of Minnesota, Twin Cities, clearly stated on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding disclosure is itself a concern."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses OpenAI-o3, GPT-5, Claude-Sonnet-4, etc. without stating their training data cutoff dates. KernelBench was published in 2025 and these models may have seen it."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether KernelBench tasks or their solutions appeared in the training data of the models used."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "KernelBench was published on arXiv in Feb 2025. The models used (o3, GPT-5) may have been trained on data including KernelBench. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports API cost ($0.30 average per kernel) and wall-clock time (26.5 minutes average) broken down by level. Figure 6 shows cost-performance tradeoff curves."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Total compute is implicitly stated: 250 tasks at ~26.5 min each on a single RTX 6000, with ~$0.30 API cost per kernel. Comparison with Agentic Baseline (6 H100 hours, $5 per kernel) is provided."
       }
     }
   },
   "claims": [
     {
       "claim": "CudaForge achieves 97.6% correctness and 1.68x average speedup over PyTorch baselines on KernelBench Levels 1-3.",
       "evidence": "Table 1 shows 97.6% correctness and 1.677x performance on 250 KernelBench tasks (Section 3.3).",
       "supported": "strong"
     },
     {
       "claim": "CudaForge outperforms Kevin-32B (RL-based) with 98.0% correctness and 1.662x speedup vs. Kevin's 82.0% and 1.10x on comparable tasks.",
       "evidence": "Section 3.4 and Figure 5 compare CudaForge on H200 against Kevin-32B's reported results on Levels 1-2.",
       "supported": "moderate"
     },
     {
       "claim": "Scaling up maximum iteration rounds to 30 improves CudaForge's performance to 2.27x average speedup.",
       "evidence": "Figure 7 and Table 1 (CudaForge-Scaling Up) show 2.265x on the D* subset (25 tasks). Section 3.7.",
       "supported": "moderate"
     },
     {
       "claim": "CudaForge generalizes across GPUs (A100, RTX 6000, 4090, 3090) and base models (o3, GPT-5, Claude-Sonnet-4, GPT-OSS-120B, QwQ-32B).",
       "evidence": "Table 4 shows consistent results across 4 GPUs. Table 5 shows results with various model combinations. All evaluated on D* (25 tasks).",
       "supported": "moderate"
     },
     {
       "claim": "CudaForge is significantly cheaper than the Agentic Baseline: $0.3 vs $5 API cost and 26.5 min vs 60 min per kernel.",
       "evidence": "Table 3 reports costs. However, the Agentic Baseline costs are taken from their paper and run on different hardware (H100).",
       "supported": "moderate"
     },
     {
       "claim": "Using a curated subset of 24 NCU metrics is more effective than using all NCU metrics.",
       "evidence": "Table 1 compares CudaForge vs CudaForge(full metrics) on D*: 1.767x vs 1.414x. Case study in Appendix B.1.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CudaForge is a training-free multi-agent framework for CUDA kernel generation that uses a Coder and Judge with hardware feedback from Nsight Compute metrics. On KernelBench (250 tasks), it achieves 97.6% correctness and 1.68x average speedup over PyTorch, outperforming RL-based methods (Kevin-32B) and direct LLM generation. The framework demonstrates generalization across GPU architectures and base models, with inference-time scaling showing continued improvement up to 30 iterations. Cost is reported at ~$0.30 and 26.5 minutes per kernel on a single RTX 6000.",
   "red_flags": [
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "All results are single-run point estimates with no error bars, confidence intervals, or variance across runs. Given LLM output stochasticity, results could vary significantly across runs."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats to validity, or scope boundary discussion, despite making broad generalization claims."
     },
     {
       "flag": "Unfair baseline comparison conditions",
       "detail": "Kevin-32B comparison uses results from their paper on different hardware/benchmark. Agentic Baseline comparison uses their reported costs on H100 vs CudaForge on RTX 6000. These are not fully controlled comparisons."
     },
     {
       "flag": "Small subset for key experiments",
       "detail": "Many ablation and generalization experiments (GPU cross-testing, model combinations, scaling) are run on D* with only 25 tasks, which may not be representative. No justification for this sample size."
     },
     {
       "flag": "Best-of-N evaluation protocol",
       "detail": "The paper reports 'the best-performing correct kernel among all candidates for each task,' which inflates reported performance compared to expected single-run behavior."
     },
     {
       "flag": "No contamination analysis",
       "detail": "KernelBench was published before the models were likely trained. No discussion of whether models may have memorized benchmark solutions."
     }
   ],
   "cited_papers": [
     {
       "title": "Kevin: Multi-turn RL for Generating CUDA Kernels",
       "authors": ["Carlo Baronio", "Pietro Marsella", "Ben Pan", "Simon Guo", "Silas Alberti"],
       "year": 2025,
       "arxiv_id": "2507.11948",
       "relevance": "RL-based approach for CUDA kernel generation, direct baseline comparison for CudaForge."
     },
     {
       "title": "Towards Robust Agentic CUDA Kernel Benchmarking, Verification, and Optimization",
       "authors": ["Robert Tjarko Lange", "Qi Sun", "Aaditya Prasad", "Maxence Faldor", "Yujin Tang", "David Ha"],
       "year": 2025,
       "arxiv_id": "2509.14279",
       "relevance": "Multi-agent agentic baseline for CUDA kernel optimization, key comparison point."
     },
     {
       "title": "KernelBench: Can LLMs Write Efficient GPU Kernels?",
       "authors": ["Anne Ouyang", "Simon Guo", "Simran Arora", "Alex L. Zhang", "William Hu", "Christopher Ré", "Azalia Mirhoseini"],
       "year": 2025,
       "arxiv_id": "2502.10517",
       "relevance": "The primary benchmark used for evaluation; assesses LLM capability for CUDA code generation."
     },
     {
       "title": "A Survey on Code Generation with LLM-based Agents",
       "authors": ["Yihong Dong", "Xue Jiang", "Jiaru Qian"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Survey of LLM-based agents for code generation, relevant to agentic programming survey scope."
     },
     {
       "title": "A Survey on Large Language Models for Code Generation",
       "authors": ["Juyong Jiang", "Fan Wang", "Jiasi Shen", "Sungju Kim", "Sunghun Kim"],
       "year": 2024,
       "arxiv_id": "2406.00515",
       "relevance": "Survey of LLMs for code generation, contextualizes LLM code capabilities."
     },
     {
       "title": "AgentVerse: Facilitating Multi-Agent Collaboration and Exploring Emergent Behaviors",
       "authors": ["Weize Chen"],
       "year": 2023,
       "arxiv_id": "2308.10848",
       "relevance": "Multi-agent collaboration framework that inspired CudaForge's prompt design."
     },
     {
       "title": "CUDA-L1: Improving CUDA Optimization via Contrastive Reinforcement Learning",
       "authors": ["DeepReinforce Team"],
       "year": 2025,
       "relevance": "RL-based CUDA optimization approach whose 'fake kernel' problem is exposed by CudaForge authors."
     },
     {
       "title": "Agent S: An Open Agentic Framework that Uses Computers Like a Human",
       "authors": ["Saaket Agashe"],
       "year": 2024,
       "arxiv_id": "2410.08164",
       "relevance": "Agentic framework for computer interaction, inspired CudaForge's agent prompt design."
     }
   ]
 }