ai-research-survey

scan.json (29558B)

---

 {
   "paper": {
     "title": "Optimizing PyTorch Inference with LLM-Based Multi-Agent Systems",
     "authors": [
       "Kirill Nagaitsev",
       "Luka Grbčić",
       "Samuel Williams",
       "Costin Iancu"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2511.16964",
     "doi": "10.48550/arXiv.2511.16964"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper presents PIKE, a logical framework for comparing multi-agent LLM-based PyTorch optimization systems. PIKE-B (exploit-heavy, branching search with error-fixing agents) achieves 2.88× geomean speedup over PyTorch Eager on H100 GPU across a refined KernelBench suite, outperforming torch.compile (1.64×), TensorRT (1.41×), and METR (1.40×). Exploit-heavy strategies paired with error-fixing agents consistently outperform explore-heavy strategies, and performance correlates with the granularity (aggressiveness) of optimization steps.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided anywhere in the paper. The PIKE implementations are described algorithmically but no source code link is given."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses KernelBench (Ouyang et al., 2025), a publicly available benchmark. The METR-refined variant is also publicly available. The authors describe their further filtering (Level 3-pike) in §4.1 and Table 1."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (H100 80GB HBM3, Intel Xeon Platinum 8480+, 40+ CPU threads) and the evaluator is containerized with Docker. However, no requirements.txt, Dockerfile, or software version list is provided — PyTorch version, CUDA version, Triton version, etc. are not stated."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The paper describes the algorithm (Algorithm 1) and hyperparameters, but there are no step-by-step instructions or scripts for replicating the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., 2.88× speedup). No confidence intervals or error bars are provided for any results, despite the stochastic nature of LLM generation (temperature 0.8)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims like 'PIKE-B outperforms PIKE-O' are based solely on comparing geomean speedup numbers without any statistical testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Speedup ratios relative to PyTorch Eager baseline are reported throughout (e.g., 2.88× for PIKE-B, 1.64× for torch.compile). These provide baseline context for the magnitude of improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark suite contains 30 Level 3-pike tasks and 14 Level 5 tasks. No justification is given for why these sample sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures across runs are reported. Each configuration appears to be run once with the 300-query budget. Given the stochastic LLM (temperature 0.8), repeated runs would likely yield different results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: PyTorch Eager (reference), torch.compile with max autotuning, TensorRT, and METR solutions (§4.2). Results are shown in Tables 3-4 and Figures 3-5."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "METR (2025) represents the state of the art for LLM-based kernel optimization. TorchInductor and TensorRT are current production compilers. All baselines are contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablations are performed: PIKE-B (no EFA, no IBA, cheap EFA) in §5.2.1, and PIKE-O variants progressively modified toward PIKE-B (mut, npar, 1isl, EO, SL) in §5.2.2. Figure 5 summarizes all ablations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported using speedup (geomean), LLM query count, monetary cost per task, SLOC, mean error-fix attempts, LoC changed per step, and cosine similarity between optimization steps (§4.5-4.7, Table 2)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant to the claims. The paper evaluates GPU kernel optimization quality through automated correctness checks and runtime measurements."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "PIKE-B hyperparameters were selected via 'initial tuning' (§4.4) but no validation/test split is described. It is unclear whether tuning was done on the same benchmark tasks used for final evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3 and 4 provide per-task speedup breakdowns for all 30 Level 3-pike and 14 Level 5 tasks across all methods. Appendix B provides detailed analysis of individual task results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B discusses task removals (LSTM/GRU, §B.1.1), MLP noise issues (§B.1.2), invalid solutions (§B.3.6 — invalid METR DeepSeek3MLA solution), and challenging CUDA techniques (§B.4). Tasks where PIKE underperforms are visible in per-task tables."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: PIKE-O (mut,npar) drops to 1.99 when parallelism is reduced (§5.2.2); PIKE-O default does not benefit from EFA (2.17 vs 2.15, §5.2.2); crossover vs mutation shows no improvement (PIKE-O mut same as default at 2.15); PIKE-B without EFA drops to 1.98 (§5.2.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims: (1) 2.88× speedup on H100 — confirmed in Table 3 geomean; (2) exploit-heavy strategies perform best with error-fixing — confirmed across §5.1-5.2; (3) performance correlates with step granularity — supported by Figures 6-7 and Table 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made via controlled ablation: removing EFA from PIKE-B drops performance from 2.88 to 1.98 (§5.2.1); progressive PIKE-O ablations isolate the effect of each component change (§5.2.2). Single-variable manipulations are adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract specifies 'on an H100 GPU across diverse tasks in KernelBench.' The paper uses a single LLM (Gemini 2.5 Pro) and a single GPU type, and does not claim general applicability beyond this setting. Results are presented specifically for KernelBench Level 3-pike and Level 5."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for its findings. For example, the PIKE vs METR comparison confounds the multi-agent strategy with the model choice (Gemini 2.5 Pro vs o1/Claude/GPT-4o). No discussion of whether Gemini 2.5 Pro's specific capabilities drive the results rather than the search strategy."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures speedup relative to PyTorch Eager on specific benchmark tasks and frames results as 'inference speedup.' The measurement matches the claim granularity — no proxy gap exists between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'Gemini 2.5 Pro' and 'Gemini 2.5 Flash' (§4.5) — marketing names without snapshot dates or API versions. Also mentions 'OpenAI's text-embedding-3-large' (§4.7) without a version. Per the schema, marketing names without API versions do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Agent roles are described conceptually (IBA generates ideas, COA optimizes code, EFA fixes errors) but no actual prompt text is provided for any agent. The reader cannot reconstruct the prompts sent to the LLM."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed hyperparameters in §4.4: temperature 0.8, PIKE-B (n=10, m=5, k=4), PIKE-O (population=25, archive=12, islands=3, explore=0.2, exploit=0.7, parallel evaluations=10). Correctness tolerance in Appendix A (atol=0.01, rtol=0.01)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent framework is described in detail in §2-3: library management, seed selection with explore/exploit ratios, mutation vs crossover, error-fixing loops, island-based genetic algorithms. Figure 1 shows the workflow, Algorithm 1 provides PIKE-B pseudocode."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark filtering is documented: §4.1 describes removing Levels 1-2, filtering LSTM/GRU from Level 3, using METR's Level 5. Table 1 summarizes the final suite. Appendix B.1 details specific task removals with rationale."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section exists. Budget constraints are mentioned in passing ('Due to the high cost of experimentation, we were unable to conduct a complete hyperparameter sweep' in §4.4) but there is no substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address threats like single-GPU generalizability, single-model dependence, lack of repeated runs, or benchmark representativeness."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what the results do NOT show — e.g., whether findings transfer to other GPUs, other LLMs, other optimization targets, or production workloads beyond KernelBench."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is released. Generated solutions, LLM responses, evaluation logs, and timing measurements are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation procedure is well-described: Docker-containerized evaluation, Triton's do_bench for timing with warmup and synchronization, torch.allclose correctness checks with specified tolerances (§4.3, §4.6, Appendix A)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from KernelBench, a standard public benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from benchmark selection through filtering, evaluation, correctness checking, timing, and metric computation is documented across §4.1-4.6. Task removal rationale is provided in Appendix B.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgements: U.S. Department of Energy, Office of Science (DE-FOA-0003264, FP00018807, DE-AC02-05CH11231) and DOE Computational Science Graduate Fellowship (DE-SC0024386)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Northwestern University and Lawrence Berkeley National Laboratory. The authors are not evaluating a product from their own employer."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The U.S. Department of Energy funds basic scientific computing research and has no financial interest in the specific outcome of PIKE vs other optimization methods."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training cutoff for Gemini 2.5 Pro/Flash is not stated. The LLM is generating code optimizations for benchmark tasks, and its training data could include prior solutions."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Gemini 2.5 Pro was trained on KernelBench tasks, prior solutions, or optimized PyTorch code that could inform its generations."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "KernelBench was published in early 2025. Gemini 2.5 Pro could have been trained on the benchmark tasks or prior solutions. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed cost analysis in §4.5 and Figures 3b/4b: $25-50 per task for 300 queries, mean cost $50.96 (PIKE-B) vs $39.59 (PIKE-O) per task (Table 2). Gemini API pricing cited. Cost-performance tradeoffs analyzed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Compute budget clearly stated: 300 LLM queries per task, H100 GPU with 80GB HBM3, 40+ CPU threads, parallel CUDA/Triton compilations. Total suite cost: $25 × 30 = $750 for Level 3-pike at $25/task budget (§4.5)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are not reported across multiple random seeds. Despite using LLM temperature 0.8, each configuration appears to be run once. No analysis of how results vary across independent optimization runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The budget is 300 queries per task, but the number of independent optimization runs is not explicitly stated. It appears each configuration was run once. The comparison with METR notes 'our single 300-query runs are compared against the best results from 4 or more METR runs.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "PIKE-B parameters were selected via 'initial tuning' (§4.4) but no search budget is reported — how many configurations were tried, what method was used, or what compute was spent on tuning."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The systematic PIKE-O ablation (§5.2.2) traces configuration choices through progressive modifications (mut → npar → 1isl → EO → SL), showing the effect of each change. PIKE-B ablation (§5.2.1) demonstrates the contribution of EFA and IBA."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many pairwise comparisons are made across configurations, levels, and budgets without any statistical tests or multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own PIKE variants and compare against METR (whose framework is not open-source). They run METR's best solutions on their hardware (fair), but do not acknowledge or discuss self-comparison bias per Lucic et al. (2018)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figures 3 and 4 explicitly plot performance (geomean speedup) as a function of both LLM queries per task and cost per task. This allows direct comparison at matched budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper critically evaluates benchmark quality: removing LSTM/GRU tasks due to pathological baselines (§B.1.1), identifying MLP noise (§B.1.2), documenting invalid METR solutions (§B.3.6), and creating a refined Level 3-pike suite. This shows active engagement with construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The comparison against METR confounds the multi-agent strategy with the LLM choice: PIKE uses Gemini 2.5 Pro while METR uses o1, Claude 3.5 Sonnet, gpt-4o, and o3-mini-high. The paper does not control for or discuss this confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. KernelBench tasks are based on well-known PyTorch architectures (ResNet, DenseNet, MinGPT, etc.) that have been extensively discussed online. The LLM could have seen optimized versions in training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The evaluation gives the LLM full access to the PyTorch source code, which is the intended setup, but no analysis of whether prior knowledge from training influences generations."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Many Level 3-pike tasks share structural similarities (multiple DenseNet variants, multiple EfficientNet variants, etc.), and solutions to one task could inform another. This non-independence is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used. No canary strings, membership inference tests, or decontamination analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "PIKE-B achieves an average 2.88× speedup on H100 GPU across Level 3-pike KernelBench tasks",
       "evidence": "Table 3 shows per-task speedups with geomean 2.88×. Figure 5(a) confirms this is the best-performing configuration at 300-query budget.",
       "supported": "strong"
     },
     {
       "claim": "Exploit-heavy strategies outperform explore-heavy strategies for PyTorch optimization",
       "evidence": "PIKE-B (exploit-heavy, 2.88×) outperforms default PIKE-O (explore-heavy, 2.17×). Progressive PIKE-O ablation toward exploitation improves from 2.17 to 2.81 (§5.2.2, Figure 5). Consistent across both Level 3-pike and Level 5.",
       "supported": "strong"
     },
     {
       "claim": "Error-fixing agents (EFA) are critical for exploit-heavy strategies but not for explore-heavy ones",
       "evidence": "PIKE-B drops from 2.88 to 1.98 without EFA (§5.2.1). PIKE-O changes negligibly from 2.17 to 2.15 without EFA (§5.2.2). Explained by PIKE-B producing riskier, larger code changes that need more fixing (Table 2, Figure 6).",
       "supported": "strong"
     },
     {
       "claim": "Performance correlates with optimization step granularity — larger steps yield better results",
       "evidence": "Table 2: PIKE-B mean SLOC 244 vs PIKE-O 169. Figures 6-7 show PIKE-B changes more LoC per step. PIKE-B achieves higher speedup despite fewer total steps (160 vs 198). Correlation between step size and performance across PIKE-O variants also shown.",
       "supported": "moderate"
     },
     {
       "claim": "PIKE-B with cheap EFA (Gemini 2.5 Flash) provides the best cost-performance tradeoff at $25/task",
       "evidence": "Figure 3(b) shows PIKE-B cheap EFA achieves 2.51× at $25/task vs 2.31× for standard PIKE-B and 2.33× for PIKE-O variant. Mean EFA query cost drops from $0.15 to $0.04 (§5.2.1).",
       "supported": "moderate"
     },
     {
       "claim": "PIKE outperforms torch.compile, TensorRT, and METR across both benchmark levels",
       "evidence": "Level 3-pike (Table 3, Figure 5a): PIKE-B 2.88× vs torch.compile 1.64×, TensorRT 1.41×, METR 1.40×. Level 5 (Table 4, Figure 5b): PIKE-B 2.57× vs torch.compile 1.29×, TensorRT 1.25×, METR 1.50×.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or error bars on a stochastic system",
       "detail": "All results are single-run point estimates despite using LLM temperature 0.8. The evolutionary search process is inherently stochastic, yet no repeated runs, confidence intervals, or variance measures are reported. Geomean speedup differences of 0.07 (e.g., PIKE-B 2.88 vs PIKE-O variant 2.81) could easily fall within run-to-run variance."
     },
     {
       "flag": "Confounded METR comparison",
       "detail": "PIKE uses Gemini 2.5 Pro exclusively while METR uses o1, Claude 3.5 Sonnet, gpt-4o, and o3-mini-high. The paper attributes PIKE's advantage to the multi-agent strategy, but the model choice is an uncontrolled confound. Additionally, METR's 'best of 4+ runs' is compared against PIKE's single run, which could either advantage or disadvantage PIKE."
     },
     {
       "flag": "No contamination analysis for benchmark tasks",
       "detail": "KernelBench tasks are based on well-known architectures (ResNet, DenseNet, MinGPT, etc.) with extensive online optimization discussions. Gemini 2.5 Pro's training data likely includes optimized CUDA/Triton implementations for these architectures, yet no contamination analysis is performed."
     },
     {
       "flag": "No code release for reproduction",
       "detail": "Neither the PIKE framework code nor generated solutions are released. The paper describes algorithms at a high level but key implementation details (actual prompts, evaluation harness specifics, filtering logic) are missing. Full reproduction is impossible."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity discussion. Significant limitations — single GPU type, single LLM, single benchmark, single run, no prompt release, incomplete hyperparameter search — are not acknowledged."
     }
   ],
   "cited_papers": [
     {
       "title": "KernelBench: Can LLMs Write Efficient GPU Kernels?",
       "authors": ["Alex Ouyang", "Simon Guo", "Simran Arora", "Alex L. Zhang", "William Hu", "Christopher Ré", "Azalia Mirhoseini"],
       "year": 2025,
       "arxiv_id": "2502.10517",
       "relevance": "The primary benchmark used for evaluation; defines the LLM-generated GPU kernel task."
     },
     {
       "title": "Measuring Automated Kernel Engineering",
       "authors": ["METR"],
       "year": 2025,
       "relevance": "Refined KernelBench with Level 5 tasks and provides the primary baseline for LLM-based kernel optimization comparison."
     },
     {
       "title": "AlphaEvolve: A Coding Agent for Scientific and Algorithmic Discovery",
       "authors": ["Alexander Novikov", "Ngân Vũ", "Marvin Eisenberger"],
       "year": 2025,
       "arxiv_id": "2506.13131",
       "relevance": "LLM-based code evolution framework that inspired OpenEvolve and PIKE-O's evolutionary approach."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Influential benchmark for LLM-based software engineering agents, cited as related evaluation framework."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversations",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2024,
       "relevance": "Multi-agent LLM framework combining outputs from different models; contextualizes PIKE in the broader multi-agent landscape."
     },
     {
       "title": "MetaGPT: Meta Programming for a Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong", "Mingchen Zhuge", "Jonathan Chen"],
       "year": 2024,
       "relevance": "Multi-agent framework splitting work among specialized agents for coding tasks; related approach to PIKE's agent specialization."
     },
     {
       "title": "The AI CUDA Engineer: Agentic CUDA Kernel Discovery, Optimization and Composition",
       "authors": ["Robert Tjarko Lange", "Aaditya Prasad", "Qi Sun"],
       "year": 2025,
       "relevance": "Agent-driven CUDA kernel optimization system using evolutionary search and LLM verification."
     },
     {
       "title": "CUDA-L1: Improving CUDA Optimization via Contrastive Reinforcement Learning",
       "authors": ["Xuan Li", "Xiaoyu Sun", "Anjiang Wang"],
       "year": 2025,
       "arxiv_id": "2507.14111",
       "relevance": "RL-based CUDA optimization with runtime feedback, alternative approach to LLM-based kernel generation."
     },
     {
       "title": "Astra: A Multi-Agent System for GPU Kernel Performance Optimization",
       "authors": ["Andrew Wei", "Tianle Sun", "Yuvanesh Seenichamy"],
       "year": 2025,
       "arxiv_id": "2509.07506",
       "relevance": "Multi-agent system for production SGLang kernel optimization, coordinating generation, testing, profiling, and planning agents."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2024,
       "relevance": "Cost-effective LLM cascade strategy relevant to PIKE's cost-performance analysis."
     },
     {
       "title": "Kevin: Multi-Turn RL for Generating CUDA Kernels",
       "authors": ["Cristian Baronio", "Pietro Marsella", "Bo Pan"],
       "year": 2025,
       "arxiv_id": "2507.11948",
       "relevance": "Multi-turn RL CUDA optimizer balancing correctness and performance, exceeding frontier models."
     },
     {
       "title": "MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation",
       "authors": ["Qian Huang", "Jian Vora", "Percy Liang", "Jure Leskovec"],
       "year": 2024,
       "relevance": "Benchmark for evaluating LLM agents on ML tasks; part of the broader agent evaluation landscape."
     }
   ]
 }