ai-research-survey

scan.json (21655B)

---

 {
   "paper": {
     "title": "The Illusion of Diminishing Returns: Measuring Long Horizon Execution in LLMs",
     "authors": ["Akshit Sinha", "Arvindh Arun", "Shashwat Goel", "Steffen Staab", "Jonas Geiping"],
     "year": 2025,
     "venue": "Preprint (arXiv)",
     "arxiv_id": "2509.09677"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper header indicates 'Code' and 'Dataset' links. The paper states the benchmark can generate new examples programmatically."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "A 'Dataset' link is indicated at the top of the paper, and examples are generated programmatically from the described procedure."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided in the paper. Section E.5 mentions compute details but not software dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the experimental setup is described in detail (Section 3 and Appendix E), the paper does not provide step-by-step reproduction instructions or scripts to replicate the main experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows 'mean ± one standard deviation over 100 samples' as shaded regions. Error bars/uncertainty bands appear in multiple figures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., larger models execute longer, thinking fixes self-conditioning) but reports no statistical significance tests. Comparisons are based on visual inspection of curves."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context, e.g., 'GPT-5 (codenamed Horizon) with 2176 steps' vs 'Claude-4 Sonnet (432 steps)', and horizon lengths are quantified for different model sizes (Figure 4b)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 100 samples per condition but does not justify this sample size or discuss whether it provides sufficient statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation is reported across 100 samples as shaded regions in Figure 4 and other plots."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple model families and sizes are compared (Qwen3 4B-32B, Gemma3 4B-27B, frontier models). A hypothetical constant step-accuracy baseline (0.99) is also included as a dotted line in Figure 4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary frontier models: GPT-5, Claude-4 Sonnet, Grok 4, Gemini 2.5 Pro, Kimi K2, DeepSeek-R1/V3."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablations are performed: with/without CoT, thinking vs non-thinking, context window sizes (Appendix A.2), majority voting (Appendix B), self-verification prompting (Appendix A.1), and error rate manipulation (Section 3.2)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are defined and used: step accuracy, turn accuracy, task accuracy, and horizon length (Section 2)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates LLM execution on a synthetic arithmetic task with deterministic ground truth. Human evaluation is not relevant to these claims."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Examples are generated programmatically on the fly; there is no training/tuning split to hold out from."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model size, model family, turn complexity K, number of turns, with/without thinking, and induced error rates."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure modes are extensively discussed: self-conditioning effect (Section 3.2), format-following failures (Appendix F), self-verification failures (Appendix A.1), and errors deconstructed into retrieval vs composition (Appendix D)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: majority voting does not match thinking (Appendix B), self-verification prompting does not fix self-conditioning (Appendix A.1), scaling model size does not mitigate self-conditioning (Result 4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about compounding returns, self-conditioning, and thinking fixing self-conditioning are all supported by corresponding experimental results (Sections 2.1, 3.2, and 3.3)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses controlled counterfactual experiments to support causal claims. The self-conditioning claim is tested by manipulating error rates in chat history (Section 3.2), isolating the causal mechanism through controlled variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states the task 'does not reflect complexities and sources of error arising in real agentic tasks,' that results are about 'pretrained LLMs, and not inherent properties of transformers,' and that 'Improvement on our task is necessary, but not sufficient for long-horizon execution on real-world tasks.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly tests alternative explanations: long-context degradation vs self-conditioning (Section 3.2), and discusses redundancy of internal circuits as an alternative explanation for scaling benefits (Section 3.1, citing Lindsey et al., 2025)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to by marketing names (GPT-5, Claude-4 Sonnet, Grok 4, Gemini 2.5 Pro, Qwen3-32B) without API version strings or snapshot dates. Appendix E.4 lists model names but not specific version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section E.2 and E.3 provide the exact prompting setup including system prompt, few-shot examples, and format instructions. The paper states 'the exact prompt' is provided in Section E."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E mentions temperature settings and Appendix Figure 13 shows results are robust to temperature. The binary search procedure for horizon length uses an 80% accuracy threshold."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The experiments are direct model queries with controlled chat history."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data generation procedure is fully documented: dictionary of 5-letter English words mapping to integers in [-99, 99], keys sampled per turn, few-shot examples provided (Section 3, Appendix E.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' paragraph appears at the end of Section 5 (Conclusion), discussing multiple specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: synthetic task does not reflect real agentic complexity, results may change with task-specific finetuning, task accuracy metric does not account for self-correction, and results are about pretrained LLMs not inherent transformer properties."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'Improvement on our task is necessary, but not sufficient for long-horizon execution on real-world tasks.' It also notes the task does not cover diverse actions, only repeated ones."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While a dataset link is indicated, the raw model outputs and per-sample results are not described as being released. Only aggregate results in figures are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data generation procedure is fully specified: dictionary construction, key sampling, integer ranges, number of samples (100), and evaluation methodology (Sections 2-3, Appendix E)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is programmatically generated from a synthetic task."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from task generation to evaluation is documented: dictionary creation, key sampling, model querying, answer extraction, and accuracy computation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section lists CHIPS JU grant No. 101140087 (SMARTY), BMBF funding number 16MEE0444, IMPRS-IS, ELLIS PhD programs, and AISA cluster compute."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Cambridge, University of Stuttgart, Max Planck Institute, ELLIS Institute Tübingen, University of Southampton, Tübingen AI Center."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are academic/government bodies (EU CHIPS JU, German BMBF, IMPRS-IS) with no financial stake in the performance of any particular LLM."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training cutoff dates are not stated for any of the evaluated models. However, the paper notes the benchmark is contamination-free since examples are generated programmatically."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses this: 'An advantage of our benchmark is that it is contamination-free, as new examples can be generated programmatically' (Section 3.3)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper argues contamination is structurally impossible because examples are generated fresh each time: 'contamination-free, as new examples can be generated programmatically' (Section 3.3)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or wall-clock times are reported for the experiments, despite querying multiple frontier API models extensively."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Appendix E.5 mentions the AISA cluster but does not quantify GPU hours, total API spend, or overall compute budget."
       }
     }
   },
   "claims": [
     {
       "claim": "Diminishing gains in single-step accuracy compound into exponential improvements in horizon length for long-horizon tasks.",
       "evidence": "Proposition 1 derives H_s(p) = ln(s)/ln(p), showing hyperbolic growth. Figure 2 illustrates this relationship. METR's empirical data on doubling horizon length every 7 months is consistent with this formula (Section 2.1).",
       "supported": "strong"
     },
     {
       "claim": "Long-horizon execution is challenging even when models have the required knowledge and plan.",
       "evidence": "Figure 4(a) shows near-perfect first-step accuracy for most models, yet task accuracy falls below 50% within 15 turns even for the best model (Section 3.1, Result 1).",
       "supported": "strong"
     },
     {
       "claim": "Scaling model size yields non-diminishing improvements in horizon length.",
       "evidence": "Figure 4(b-c) shows larger models sustain higher task accuracy for more turns across Qwen3 and Gemma3 families. Only 4 sizes per family are tested (Section 3.1, Result 2).",
       "supported": "moderate"
     },
     {
       "claim": "Models self-condition on their own errors, causing per-step accuracy to degrade beyond what long-context degradation alone explains.",
       "evidence": "Counterfactual experiment manipulating error rates in chat history (Section 3.2). Figure 5 shows accuracy at turn 100 consistently degrades as induced error rate increases, even with error-free history showing some long-context degradation.",
       "supported": "strong"
     },
     {
       "claim": "Thinking (RL-trained reasoning) fixes the self-conditioning effect.",
       "evidence": "Figure 6 shows Qwen3 thinking models maintain stable accuracy at turn 100 regardless of induced error rate in context (Section 3.2, Result 5).",
       "supported": "strong"
     },
     {
       "claim": "GPT-5 can execute over 2100 steps in a single turn, far ahead of Claude-4 Sonnet (432) and Grok 4 (384).",
       "evidence": "Figure 7(b) benchmarks frontier thinking models on single-turn execution length with binary search for K at ≥80% accuracy (Section 3.3, Result 7).",
       "supported": "moderate"
     },
     {
       "claim": "Parallel test-time compute (majority voting) cannot match sequential compute (thinking) for long-horizon execution.",
       "evidence": "Appendix B, Figure 10 shows majority voting with N=100 provides only marginal improvement over the base model and does not match CoT performance.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper demonstrates that diminishing returns in single-step LLM accuracy compound into exponential gains in the length of tasks models can complete, following a hyperbolic relationship. It identifies a 'self-conditioning' effect where models become more error-prone when their context contains prior mistakes, distinct from long-context degradation. Thinking (RL-trained reasoning) eliminates self-conditioning and dramatically extends single-turn execution length, with GPT-5 executing 2176 steps vs 432 for Claude-4 Sonnet. Sequential test-time compute significantly outperforms parallel scaling (majority voting) for long-horizon execution.",
   "red_flags": [
     {
       "flag": "Synthetic task generalizability",
       "detail": "The key-value dictionary addition task is extremely simple and repetitive. The paper acknowledges this limitation but the title and framing ('The Illusion of Diminishing Returns') make broad claims about LLM scaling value that extend well beyond what this synthetic task demonstrates."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Despite making multiple comparative claims about model scaling and self-conditioning effects, no statistical significance tests are reported. Conclusions are drawn from visual inspection of curves."
     },
     {
       "flag": "Model versions unspecified",
       "detail": "Frontier models (GPT-5, Claude-4 Sonnet, Grok 4, etc.) are referenced by marketing names without API versions or snapshot dates, making results non-reproducible as these models update."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring AI ability to complete long tasks",
       "authors": ["Thomas Kwa", "Ben West", "Joel Becker"],
       "year": 2025,
       "arxiv_id": "2503.14499",
       "relevance": "METR benchmark for long-horizon agentic task completion, directly motivates this paper's framing of horizon length."
     },
     {
       "title": "The illusion of thinking: Understanding the strengths and limitations of reasoning models via the lens of problem complexity",
       "authors": ["Parshin Shojaee"],
       "year": 2025,
       "arxiv_id": "2506.06941",
       "relevance": "Claims thinking models give an 'illusion of thinking'; this paper directly responds by arguing failures are execution, not reasoning."
     },
     {
       "title": "Do large language model benchmarks test reliability?",
       "authors": ["Joshua Vendrow"],
       "year": 2025,
       "arxiv_id": "2502.03461",
       "relevance": "Evaluates LLM reliability on benchmarks, complementary to this paper's focus on long-horizon reliability."
     },
     {
       "title": "Scaling LLM test-time compute optimally can be more effective than scaling model parameters",
       "authors": ["Charlie Snell"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Studies test-time compute scaling; this paper contrasts parallel vs sequential compute for execution tasks."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao"],
       "year": 2023,
       "relevance": "Foundational work on reasoning-before-acting paradigm for LLM agents, cited to motivate thinking benefits."
     },
     {
       "title": "Navigating the jagged technological frontier: Field experimental evidence of the effects of AI on knowledge worker productivity and quality",
       "authors": ["Fabrizio Dell'Acqua"],
       "year": 2023,
       "relevance": "RCT on AI productivity impact; coined 'jagged frontier' concept referenced in this paper."
     },
     {
       "title": "tau-bench: A benchmark for tool-agent-user interaction in real-world domains",
       "authors": ["Shunyu Yao"],
       "year": 2024,
       "arxiv_id": "2406.12045",
       "relevance": "Benchmark for evaluating LLM agent reliability in tool-use scenarios."
     },
     {
       "title": "AI Scientists Fail Without Strong Implementation Capability",
       "authors": ["Minjun Zhu"],
       "year": 2025,
       "arxiv_id": "2506.01372",
       "relevance": "Shows LLM execution failures in scientific coding tasks, complementary evidence for execution as bottleneck."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws paper; this work extends the discussion to long-horizon task completion."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling laws; this paper argues step-level diminishing returns mask horizon-level gains."
     }
   ]
 }