ai-research-survey

scan.json (19860B)

---

 {
   "paper": {
     "title": "The Cost of Dynamic Reasoning: Demystifying AI Agents and Test-Time Scaling from an AI Infrastructure Perspective",
     "authors": ["Jiin Kim", "Byeongjun Shin", "Jinha Chung", "Minsoo Rhu"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2506.04301"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Open-sourced at https://github.com/VIA-Research/AgentBench' in Section I, providing a repository URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: HotpotQA, WebShop, MATH, HumanEval, and ShareGPT, all of which are publicly available datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section III specifies vLLM version 0.6.6, PyTorch 2.6, CUDA 12.8, GCP instance types (a2-highgpu-1g, a2-highgpu-8g), NVIDIA A100 40GB GPUs, and specific CPU/memory configurations."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the GitHub repo is linked and hardware is specified, the paper does not include step-by-step reproduction instructions or describe how to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., accuracy percentages, latency values) without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares agents and configurations by raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports relative increases with baselines, e.g., '62.1x-136.5x increase in GPU energy per query' compared to ShareGPT, and percentage improvements like '60.1% reduction in prefill latency'."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section V states 'we used a benchmark of 50 sample questions' without justifying why 50 was chosen or discussing whether this is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "While latency distributions are shown (Figure 7) and 95th percentile latencies are reported, no standard deviations or variance measures across experimental runs are provided. Results appear to be single-run."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "CoT serves as a non-agentic baseline, and ShareGPT serves as the conventional single-turn LLM inference baseline. Multiple agents are compared against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The agents evaluated (ReAct, Reflexion, LATS, LLMCompiler) are well-established and representative of current agent design patterns."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section V analyzes the impact of individual design parameters: iteration budget (Figure 14), few-shot examples (Figure 15), sequential vs. parallel scaling (Figure 16), and model size (Figure 17)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports accuracy, latency (average and 95th percentile), GPU energy consumption (Wh), token counts, GPU utilization, KV cache memory, and accuracy-per-latency ratio."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a systems characterization paper measuring computational costs; human evaluation of outputs is not relevant to the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses established benchmark test sets (HotpotQA, WebShop, MATH, HumanEval) with their official evaluation protocols."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (HotpotQA, WebShop, MATH, HumanEval), per agent type, and per component (prefill vs. decode, LLM vs. tool latency)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses diminishing returns, outlier tasks that consume full iteration budgets, cases where LLMCompiler's DAG-style planning leads to unnecessary tool invocations on WebShop, and accuracy decline from excessive few-shot examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that LLMCompiler performs worse than ReAct on WebShop, that excessive few-shot examples can decrease accuracy, and that sequential scaling with smaller models shows limited improvement."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about diminishing returns, widening latency variance, and unsustainable infrastructure costs are all supported by quantitative results in Sections IV-VI."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'prefix caching reduces prefill latency' are supported by controlled comparisons (with/without prefix caching). Claims about iteration budget effects are supported by systematic parameter variation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses only Llama-3.1-Instruct (8B and 70B) but makes broad claims about 'AI agents' and 'test-time scaling' generally. The datacenter power projections extrapolate from a single model family to industry-wide estimates without caveating model-specific limitations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings, such as whether different serving frameworks, batching strategies, or model architectures would substantially change the conclusions."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section III specifies 'Llama-3.1-8B-Instruct' and 'Llama-3.1-70B-Instruct' with references [44] and [45]."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompt components (instruction, few-shot, user tokens) but does not provide the actual prompt text used for any agent configuration."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or other LLM sampling parameters are reported. The paper does not state what generation settings were used."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section II-B and III describe each agent's workflow in detail: CoT, ReAct, Reflexion, LATS (tree search), and LLMCompiler (DAG planning with async execution). Figure 3 shows execution timelines."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section V mentions '50 sample questions' but does not describe how these were selected from the full benchmarks or whether any filtering was applied."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII (Discussion) discusses limitations including the lack of SLA constraints, the focus on fundamental optimizations only, and acknowledges that batching is not accounted for in energy estimates."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The discussion mentions general future directions but does not identify specific threats to validity of the study's findings, such as the small sample size (50 questions), single model family, or potential non-representativeness of selected agents."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states that LLM request batching is not accounted for, that only GPU energy is measured (excluding CPU, memory, cooling), and that the analysis uses a single model family. Section VII notes SLA analysis is left as future work."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw measurement data (latency logs, energy readings, per-query results) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III describes the measurement setup: GCP instances, vLLM serving, NVIDIA DCGM for GPU utilization, Poisson arrival distribution for traffic simulation."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data sources are standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document how the 50 sample questions were drawn from each benchmark or how measurements were aggregated into the reported statistics."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section lists IITP grants funded by the Korea government (MSIT) and Samsung Research Funding Center."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with KAIST, clearly stated on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (Korean government IITP grants and Samsung Research) do not have a direct financial stake in showing that AI agents are costly or sustainable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama-3.1-Instruct models evaluated on HumanEval and MATH but does not state the training data cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (published 2021) and MATH are used with Llama-3.1 (2024) without discussing potential training data overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval, MATH, or HotpotQA appeared in Llama-3.1's training data, despite these being well-known public benchmarks predating the model."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "This is the core contribution of the paper. Table III reports energy per query (Wh), latency per query, and Section VI projects datacenter-wide power demands."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies hardware used (A100 GPUs on GCP), reports GPU energy consumption per query, and provides detailed compute cost analysis across all configurations."
       }
     }
   },
   "claims": [
     {
       "claim": "AI agents consume 62.1x to 136.5x more GPU energy per query than conventional single-turn LLM inference.",
       "evidence": "Table III shows Reflexion consumes 130.9x (8B) and 136.5x (70B) more energy, LATS consumes 71.7x (8B) and 62.1x (70B) more energy compared to ShareGPT baseline.",
       "supported": "strong"
     },
     {
       "claim": "Prefix caching achieves an average 5.62x throughput increase for ReAct-based agent workloads versus only 1.03x for chatbot workloads.",
       "evidence": "Figure 11 and surrounding text in Section IV-C report these throughput improvements.",
       "supported": "strong"
     },
     {
       "claim": "Test-time scaling yields diminishing returns: achieving the same 4% accuracy gain costs 31x more compute at higher scaling levels.",
       "evidence": "Section V-B reports that in Reflexion, the same accuracy improvement costs 31x more latency at higher scaling points (from 56.0s requiring 269.5s additional vs. earlier 16.9s to 25.6s).",
       "supported": "strong"
     },
     {
       "claim": "Agentic workloads at current ChatGPT-scale traffic would require gigawatt-scale datacenter power.",
       "evidence": "Table IV shows 70B Reflexion at 71.4M queries/day requires ~1 GW. The paper notes this aligns with OpenAI's planned Stargate facility.",
       "supported": "moderate"
     },
     {
       "claim": "Smaller models (8B) with parallel scaling (LATS) can approach the accuracy of larger models (70B) with lower energy cost.",
       "evidence": "Figure 17 shows 8B LATS with parallel scaling achieves 80% accuracy on HotpotQA vs 82% for 70B, while consuming less energy per query.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "AI agents consume 62-137x more GPU energy per query than single-turn LLM inference, with test-time scaling exhibiting sharply diminishing returns in accuracy. Prefix caching is particularly effective for agentic workloads, achieving 5.62x throughput improvements versus 1.03x for chatbots. The paper projects that scaling agentic workloads to current ChatGPT traffic levels would require gigawatt-scale datacenter power, comparable to the announced budget for OpenAI's Stargate facility. Smaller models with parallel scaling strategies can approach larger model accuracy at lower energy cost.",
   "red_flags": [
     {
       "flag": "Very small evaluation sample",
       "detail": "All accuracy and cost-efficiency results in Section V are based on only 50 sample questions per benchmark, which is small for drawing the broad conclusions presented."
     },
     {
       "flag": "Single model family",
       "detail": "All experiments use only Llama-3.1-Instruct (8B and 70B). Results may not generalize to other architectures (e.g., GPT-4, Claude, Gemini) that may have different inference characteristics."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "No error bars, confidence intervals, or multi-run variance reported for any results. It is unclear whether results are from single runs."
     },
     {
       "flag": "Datacenter projections extrapolate heavily",
       "detail": "Power demand projections in Table IV and Section VI extrapolate single-query energy measurements to billions of queries without accounting for batching, hardware improvements, or efficiency optimizations that would apply at scale."
     }
   ],
   "cited_papers": [
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["S. Yao"],
       "year": 2022,
       "relevance": "Foundational agent framework evaluated in this paper; key baseline for agentic reasoning cost analysis."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["N. Shinn"],
       "year": 2023,
       "relevance": "Reflective agent framework evaluated as a sequential test-time scaling strategy."
     },
     {
       "title": "Language Agent Tree Search Unifies Reasoning Acting and Planning in Language Models",
       "authors": ["A. Zhou"],
       "year": 2023,
       "relevance": "Tree-search agent framework evaluated for parallel test-time scaling characteristics."
     },
     {
       "title": "An LLM Compiler for Parallel Function Calling",
       "authors": ["S. Kim"],
       "year": 2023,
       "relevance": "DAG-based planning agent evaluated for structured planning efficiency."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen"],
       "year": 2021,
       "relevance": "HumanEval benchmark used for code generation evaluation in the study."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["J. Wei"],
       "year": 2022,
       "relevance": "CoT prompting serves as the static reasoning baseline in this infrastructure analysis."
     },
     {
       "title": "The growing energy footprint of artificial intelligence",
       "authors": ["A. de Vries"],
       "year": 2023,
       "relevance": "Provides context on AI energy consumption that motivates this work's infrastructure analysis."
     },
     {
       "title": "Small language models are the future of agentic AI",
       "authors": ["P. Belcak"],
       "year": 2025,
       "relevance": "Advocates heterogeneous SLM/LLM multi-agent systems for cost reduction, aligned with this paper's findings."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversation",
       "authors": ["Q. Wu"],
       "year": 2023,
       "relevance": "Multi-agent framework relevant to understanding scaling costs of agent coordination."
     },
     {
       "title": "Scaling LLM Test-Time Compute Optimally can be More Effective than Scaling Model Parameters",
       "authors": ["C. Snell"],
       "year": 2024,
       "relevance": "Foundational work on test-time scaling that this paper extends with infrastructure cost analysis."
     }
   ]
 }