ai-research-survey

scan.json (28365B)

---

 {
   "paper": {
     "title": "Benchmark Test-Time Scaling of General LLM Agents",
     "authors": [
       "Xiaochuan Li",
       "Ryan Ming",
       "Pranav Setlur",
       "Abhijay Paladugu",
       "Andy Tang",
       "Hao Kang",
       "Shuai Shao",
       "Rong Jin",
       "Chenyan Xiong"
     ],
     "year": 2026,
     "venue": "arXiv preprint",
     "arxiv_id": "2602.18998"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Code is publicly available at https://github.com/cxcscmu/General-AgentBench.' A working URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is constructed from publicly available datasets (SWE-Bench Verified, BrowseComp, WebVoyager, MathHay, Tau2-Bench, MCP-Bench). Table 1 lists the original and sampled sizes. The GitHub repo is provided for access."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile for the evaluation framework, conda environment, or detailed library version listing is provided in the paper. The paper mentions Docker-based environments for coding tasks but does not provide environment setup specifications for reproducing the benchmark framework itself."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper itself. Appendix D describes implementation details (tool management, Docker bridge mode, evaluators) but does not provide a README-style guide with specific commands to reproduce the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Table 2 and throughout the paper are reported as point estimates (e.g., '48.0' accuracy) with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'Claude Sonnet 4.5 achieves the strongest overall performance,' 'most other models experience performance drops of approximately 30%') without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative performance changes with baselines throughout. For example, Table 3 reports baseline vs. general setting scores with percentage changes (e.g., 'Claude Sonnet 4.5: -0.2% average degradation'). Figure 4 reports mean relative degradation percentages per model. These provide context for the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark samples subsets from each original dataset (e.g., 50 from SWE-Bench Verified's 500, 124 from BrowseComp's 1266) as shown in Table 1. No justification is given for why these specific sample sizes were chosen or whether they are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variances, or spread measures are reported across experimental runs. Results appear to be single-run numbers. The parallel scaling experiments sample K=1 to K=4 trajectories but do not report variance across multiple repetitions of the entire experiment."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares general-agent performance against domain-specific baseline performance for each model (Table 3, Figures 3-4). It also compares ten different LLM agents against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models are highly contemporary: GPT-5, Claude Sonnet 4.5, Claude Haiku 4.5, Gemini 2.5-Pro, Gemini 2.5-Flash, DeepSeek-R1, DeepSeek-V3.2, Qwen3-235B, Qwen3-Next. These represent frontier models as of early 2026."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include ablation studies of its benchmark design choices (e.g., effect of tool pool size, impact of unified vs. domain-specific policies, impact of sampling strategy). The attention analysis in Appendix B compares two model architectures but is not an ablation of the benchmark or methodology."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: task accuracy/success rate, pass@K for parallel scaling, self-choice accuracy (point-wise and pair-wise), relative performance degradation from baseline, and per-domain breakdowns across four domains."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of agent outputs is included. All evaluation is automated through benchmark-specific evaluators (test suites, LLM-based semantic equivalence, DB state matching). Given claims about agent quality and robustness in real-world settings, human evaluation of trajectories or outputs would strengthen the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark samples from existing test sets (SWE-Bench Verified, BrowseComp, etc.) that serve as held-out evaluation data. The models are not tuned on this data; they are evaluated zero-shot via API calls."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-benchmark and per-domain breakdowns (Search, Code, Reason, Tool-use). Figures 3 and 5 show per-domain performance changes and scaling behaviors. Table 3 gives detailed per-benchmark scores."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses failure modes in sequential scaling (stagnant fluctuation, saturation and degradation). Figure 6 shows instance-level correctness dynamics. Section 4.3 discusses the verification gap as a failure mode. Section 3.2 includes a case study contrasting successful and less effective tool usage strategies."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central findings are largely negative: test-time scaling does not yield effective improvements due to context ceiling (sequential) and verification gap (parallel). Section 4.3 reports that GPT-5 as an external verifier 'generally underperforms models' own self-judgment,' a negative finding about external verification."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims substantial performance degradation when moving to general-agent setting (supported by Table 2, Table 3, Figure 4), context ceiling in sequential scaling (supported by Figures 5, 6, 7), and verification gap in parallel scaling (supported by Figures 5, 8). All abstract claims have corresponding evidence in the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'the accumulated history eventually overwhelms the agent's reasoning capacity, leading to instability' (Section 1) and attributes performance degradation to the unified tool interface. These causal explanations are not adequately tested through controlled experiments. The performance drop could stem from multiple confounding factors (longer prompts, different tool descriptions, different policies) that are not isolated."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to benchmark 'General LLM Agents' but tests on a specific subset of tasks sampled from seven benchmarks across four domains. The paper does not bound its generalizations to the specific tasks tested. The claim that this 'more closely reflects real-world user interactions' (Section 2.2) is not adequately bounded -- real-world usage includes many more domains and interaction patterns."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for the observed performance degradation. For example, the drop from domain-specific to general settings could be due to prompt differences, tool description length consuming context, or policy differences rather than the 'general agent' framing. The verification gap could reflect prompt engineering quality rather than fundamental model limitations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses marketing names without specific API versions or snapshot dates: 'Claude Sonnet 4.5', 'GPT-5', 'Gemini 2.5-Pro', 'Gemini 2.5-Flash'. No API version strings (e.g., 'gpt-5-2025-08-01') or snapshot dates are provided. Section 2.3 says they 'access these models via Amazon Bedrock and the Hugging Face Inference API' but does not specify API versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E provides the full universal agent system prompt, point-wise self-choice system prompt, and pair-wise self-choice system prompt. The templates include placeholders that are described (e.g., '{{Task Description}}', '{{Trajectory}}'). Section 2.2 notes 'Detailed prompt templates, tool specifications, and the unified policy are deferred to the Appendix E.'"
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 2.3 states 'we fix the decoding temperature to 0.7' but does not report other important hyperparameters such as top-p, max tokens, or stop sequences. Temperature alone is insufficient."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 and Appendix D describe the agentic scaffolding in detail: the MCP-based Host-Client-Server architecture, tool routing, interaction flow, sequential scaling via injecting additional feedback rounds, and parallel scaling via independent trajectory sampling. Figure 2 provides a framework diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 shows original and sampled sizes (e.g., BrowseComp: 1266 original, 124 sampled) but does not describe the sampling procedure or criteria used to select the subsets. No filtering criteria are stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has an 'Impact Statement' section (after Section 6) that briefly mentions risks ('may favor computationally intensive or proprietary models,' 'unreliable self-choice under parallel scaling') but there is no dedicated limitations or threats-to-validity section with substantive methodological discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Impact Statement contains only generic risk statements, not specific threats to the validity of the study's conclusions. No specific methodological limitations are discussed (e.g., small sample sizes per benchmark, single-run results, potential confounds in the domain-specific vs. general comparison)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It positions General AgentBench as 'an initial step toward evaluating general-purpose agents' (Section 2.1) but does not bound its claims to the tested domains, models, or configurations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw evaluation logs, execution traces, or per-instance results are released. The paper shows aggregate numbers and selected visualizations (e.g., Figure 6 shows 10 sampled instances) but does not provide the full underlying data for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 and Appendix A.1 describe each data source in detail: the seven benchmarks used, their domains, what they test, and how they are structured. Table 1 shows the composition. Appendix D describes the execution process and evaluation procedures."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study evaluates LLM agents on benchmarks, not human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the framework architecture is well-described (Appendix D), the data pipeline from raw benchmark data to final reported results is not fully documented. The sampling procedure from original benchmarks to the subsets used (Table 1) is not described. The paper goes from '1266 BrowseComp examples' to '124 sampled' without explaining the selection."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments section is present. The authors are from Carnegie Mellon University and Meta, but no specific funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: 'Language Technologies Institute, School of Computer Science, Carnegie Mellon University' and 'Meta.' The paper notes 'All experiments, data collection, and processing activities were conducted by Carnegie Mellon University. Meta was involved solely in an advisory role.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Meta is listed as a co-affiliation and the paper evaluates models that compete with Meta's offerings. No disclosure is made about Meta's financial interest in the outcomes. The paper does not evaluate Meta's own models (Llama), which could also indicate selection bias."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the ten evaluated models, despite using benchmarks like SWE-Bench Verified and WebVoyager that have been publicly available since 2023-2024."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark tasks could have appeared in the training data of the evaluated models. SWE-Bench issues are from public GitHub repositories; BrowseComp and WebVoyager tasks are also public."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Several of the included benchmarks (SWE-Bench, WebVoyager) were published in 2023-2024, well before the training cutoffs of models like GPT-5 and Claude Sonnet 4.5. No discussion of contamination risk is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides detailed cost tables (Tables 6-9) with per-model, per-domain, and per-scaling-setting cost breakdowns in USD. Table 6 lists unit API prices per 1M tokens. Total costs are reported for parallel scaling ($29,576), sequential scaling ($24,392), and general setting ($7,768)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides the total computational budget across all three evaluation settings. The total API cost is approximately $61,736 across all experiments. Token counts and pricing formulas are also provided."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM agents experience substantial performance degradation when moving from domain-specific to general-agent evaluation, with average relative drops ranging from 10% to 30%.",
       "evidence": "Table 3 and Figure 4 show per-model degradation. Claude Sonnet 4.5 shows only -0.2% while Gemini 2.5-Flash shows -31.2%. Detailed per-domain breakdowns in Figure 3 and Table 3.",
       "supported": "moderate"
     },
     {
       "claim": "Claude Sonnet 4.5 exhibits the strongest robustness with only 0.2% average degradation when moving to the general-agent setting.",
       "evidence": "Table 3 reports Claude Sonnet 4.5 baseline average of 45.1 vs general average of 45.0, yielding -0.2% relative change. Some domains show gains (+33.0% Search, +12.5% Reason) while others show drops (-1.4% Code, -16.8% Tool-Call).",
       "supported": "moderate"
     },
     {
       "claim": "Sequential scaling is bounded by a 'context ceiling' beyond which performance fluctuates or degrades.",
       "evidence": "Figure 5 (bottom) and Figure 7 show performance plateauing or degrading after context exceeds ~96K-112K tokens. Figure 6 shows instance-level oscillation. Results shown for 5 models across 4 domains.",
       "supported": "moderate"
     },
     {
       "claim": "Parallel scaling reveals a 'verification gap' where models cannot reliably select correct solutions from their sampled trajectories.",
       "evidence": "Figure 8 shows consistent gap between pass@K and self-choice accuracy across all models and domains. GPT-5 as external verifier also fails to close the gap, sometimes underperforming self-judgment.",
       "supported": "strong"
     },
     {
       "claim": "Increasing parallel sampling K from 1 to 4 yields an average improvement of roughly 50% in pass@K.",
       "evidence": "Figure 5 (top) shows monotonic increase in pass@K. Specific numbers cited for DeepSeek-V3.2 approaching twofold improvement in coding and reasoning domains.",
       "supported": "moderate"
     },
     {
       "claim": "Performance on static long-context benchmarks exhibits weak correlation with agentic performance.",
       "evidence": "Figure 9 shows Pearson correlations between LongBench, HELMET, MRCR and four agentic domains. Most correlations are weak (0.04-0.71), with the exception of reasoning-MRCR (0.71).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "General AgentBench reveals that LLM agents suffer 10-30% performance degradation when transitioning from domain-specific to general-agent evaluation settings, with Claude Sonnet 4.5 being notably robust (-0.2%). Sequential test-time scaling hits a context ceiling where extended interaction histories lead to stagnation or degradation rather than improvement. Parallel test-time scaling increases the theoretical solution space (pass@K) but a persistent verification gap prevents models from reliably selecting correct solutions, limiting practical gains. Static long-context benchmark performance shows weak correlation with agentic task performance.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparisons between models and settings are made on point estimates without confidence intervals, significance tests, or multiple-run variance. With small sample sizes per benchmark (50-124 examples) and single runs, observed differences may not be statistically significant."
     },
     {
       "flag": "Unexplained benchmark sampling",
       "detail": "Table 1 shows substantial subsampling from original benchmarks (e.g., 50 from 500 SWE-Bench instances, 124 from 1266 BrowseComp) without explaining the sampling criteria. If the subset is non-representative, it could bias results."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section. The Impact Statement mentions only deployment-level risks, not methodological limitations of the study design."
     },
     {
       "flag": "Confounded comparison between domain-specific and general settings",
       "detail": "The baseline vs. general comparison conflates multiple variables: tool pool size, prompt differences, context length, and domain isolation. The performance drop attributed to 'general agent' setting could be due to any of these factors, which are not independently controlled."
     },
     {
       "flag": "Missing contamination analysis",
       "detail": "Several benchmarks (SWE-Bench, WebVoyager) were published in 2023-2024, well before the training cutoffs of the evaluated 2025-2026 frontier models. No contamination discussion is provided despite this being a benchmark paper."
     },
     {
       "flag": "Model version opacity",
       "detail": "Models are identified by marketing names (GPT-5, Claude Sonnet 4.5) without API version strings or snapshot dates. Model behavior can change across API versions, making exact reproduction difficult."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig", "S. Yao", "K. Pei", "O. Press", "K. Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Foundational benchmark for evaluating LLM coding agents on real-world software engineering tasks."
     },
     {
       "title": "SWE-bench+: Enhanced Coding Benchmark for LLMs",
       "authors": ["R. Aleithan"],
       "year": 2024,
       "arxiv_id": "2410.06992",
       "relevance": "Extended version of SWE-bench addressing issues in the original benchmark's evaluation methodology."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["J. Yang", "C. E. Jimenez", "A. Wettig", "K. Lieret", "S. Yao", "K. Narasimhan", "O. Press"],
       "year": 2024,
       "relevance": "Agent framework for software engineering that defines agent-computer interfaces for code editing tasks."
     },
     {
       "title": "WebVoyager: Building an End-to-End Web Agent with Large Multimodal Models",
       "authors": ["H. He", "W. Yao", "K. Ma", "W. Yu", "Y. Dai", "H. Zhang", "Z. Lan", "D. Yu"],
       "year": 2024,
       "arxiv_id": "2401.13919",
       "relevance": "Web navigation benchmark used as one of the component benchmarks in General AgentBench."
     },
     {
       "title": "BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents",
       "authors": ["J. Wei", "Z. Sun", "S. Papay", "S. McKinney", "J. Han", "I. Fulford"],
       "year": 2025,
       "arxiv_id": "2504.12516",
       "relevance": "Search benchmark requiring multi-step web investigation, used as a component of General AgentBench."
     },
     {
       "title": "tau2-bench: Evaluating Conversational Agents in a Dual-Control Environment",
       "authors": ["V. Barres", "H. Dong", "S. Ray", "X. Si", "K. Narasimhan"],
       "year": 2025,
       "arxiv_id": "2506.07982",
       "relevance": "Tool-use benchmark for conversational agents in service scenarios, used as a component benchmark."
     },
     {
       "title": "MCP-Bench: Benchmarking Tool-Using LLM Agents with Complex Real-World Tasks via MCP Servers",
       "authors": ["Z. Wang", "Q. Chang", "H. Patel"],
       "year": 2025,
       "arxiv_id": "2508.20453",
       "relevance": "Benchmark for MCP-based tool-use evaluation, directly integrated into General AgentBench."
     },
     {
       "title": "Scaling LLM Test-Time Compute Optimally Can Be More Effective than Scaling Model Parameters",
       "authors": ["C. Snell", "J. Lee", "K. Xu", "A. Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Foundational study on test-time compute scaling that this paper extends to agentic settings."
     },
     {
       "title": "Large Language Monkeys: Scaling Inference Compute with Repeated Sampling",
       "authors": ["B. Brown", "J. Juravsky", "R. Ehrlich", "R. Clark", "Q. V. Le", "C. Re", "A. Mirhoseini"],
       "year": 2024,
       "arxiv_id": "2407.21787",
       "relevance": "Studies repeated sampling as a test-time scaling strategy, directly relevant to this paper's parallel scaling analysis."
     },
     {
       "title": "s1: Simple Test-Time Scaling",
       "authors": ["N. Muennighoff", "Z. Yang", "W. Shi", "X. L. Li"],
       "year": 2025,
       "relevance": "Recent work on test-time scaling in non-agentic settings that this paper contrasts with agentic findings."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["D. Guo", "D. Yang", "H. Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "One of the frontier reasoning models evaluated in the benchmark, representing internal scaling approaches."
     },
     {
       "title": "GAIA: A Benchmark for General AI Assistants",
       "authors": ["G. Mialon", "C. Fourrier", "C. Swift", "T. Wolf", "Y. LeCun", "T. Scialom"],
       "year": 2023,
       "arxiv_id": "2311.12983",
       "relevance": "Related general-purpose agent benchmark that evaluates multi-step reasoning and tool use."
     },
     {
       "title": "Language Agent Tree Search Unifies Reasoning, Acting, and Planning in Language Models",
       "authors": ["A. Zhou", "K. Yan", "M. Shlapentokh-Rothman", "H. Wang", "Y.-X. Wang"],
       "year": 2023,
       "arxiv_id": "2310.04406",
       "relevance": "MCTS-based agent planning approach representing more sophisticated test-time scaling beyond the basic strategies studied here."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["V. Barres"],
       "year": 2025,
       "relevance": "Related multi-task agent benchmark that evaluates LLMs across multiple task categories."
     }
   ]
 }