ai-research-survey

scan.json (28188B)

---

 {
   "paper": {
     "title": "From Task Solving to Robust Real-World Adaptation in LLM Agents",
     "authors": [
       "Pouya Pezeshkpour",
       "Estevam Hruschka"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.02760"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLM agents show a consistent gap between nominal task-solving and deployment-like robustness when tested under partial observability, noisy sensing, non-stationarity, and agent-state drift in a grid-based game. Performance degrades with grid size, but model rankings are unstable across regimes—weaker models can outperform stronger ones when strategy matches the uncertainty. Despite no explicit instruction, agents exhibit distinct trade-offs between completion, efficiency, and penalty avoidance, and action-frequency profiles reveal systematic strategy differences (e.g., GPT-5 MINI front-loads sensing, Qwen3 uses myopic trial-and-error).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository is provided: https://github.com/megagonlabs/wildgrid (footnote 1 in the abstract)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is procedurally generated from random seeds with documented parameters. The released code repository enables regeneration of all game instances."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup is mentioned in the paper. No library versions are specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The experimental setup is described (Section 3) but no commands or scripts to reproduce results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only point estimates for Accuracy, Score, and Steps. No confidence intervals or error bars are reported anywhere. Ablation figures (Figure 4) also show curves without error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Comparative claims (e.g., 'Gemini-3 Pro attains the highest accuracy on 6×6 and 10×10') are made by comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw accuracy, score, and step values are reported in Table 1. No formal effect sizes (Cohen's d, odds ratios, or contextualized percentage improvements) are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "50 instances per grid size for main evaluation and only 5 instances per data point for ablations are used, with no justification for these choices and no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, interquartile ranges, or any spread measures are reported. Table 1 shows averages only. The 50-episode results have no variance quantification."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five state-of-the-art LLMs are compared against each other (GPT-5.2, GPT-5 mini, Gemini-3 Pro, Gemini-3 Flash, Qwen3-235B), providing mutual baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All five models are current state-of-the-art: GPT-5.2, GPT-5 mini (Singh et al., 2025), Gemini-3 Pro/Flash (Comanici et al., 2025), and Qwen3-235B (Yang et al., 2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 presents single-stressor ablations: 'we deactivate all perturbations and vary only the single factor under study, to isolate its causal impact on performance.' Four factors are swept: Noise, Latent, Hazard-Spread, Teleport-Step."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: success rate (Acc), average Score, and Steps per grid size (Table 1). The paper explicitly notes these capture different aspects of performance."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated through the grid game's built-in metrics (success/failure, score, steps)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Game instances are randomly generated for each evaluation. No training or tuning is performed on these instances—all models are evaluated zero-shot, so every instance is effectively held-out."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by grid size (6×6, 8×8, 10×10) in Table 1 and by individual stressor in the ablation studies (Figure 4). Per-model action profiles are also shown (Figure 3)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses Qwen3's myopic trial-and-error behavior, GPT-5.2's score degradation from miscalibrated interaction, and specific failure drivers identified through behavioral traces and feature attribution (Section 4.4)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Qwen3 'largely fails across grid sizes' (Section 4.1). The ablations show counterintuitive results: moderate noise can improve performance (Section 4.3), and teleports can help or hurt depending on frequency. GPT-5.2's score degrades sharply with grid size."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'large gaps between nominal task-solving and deployment-like robustness' (Table 1 shows degradation), 'rankings are unstable' (different leaders per grid size), 'agents trade off completion, efficiency, and penalty avoidance' (Score/Step variation in Table 1), 'model-specific sensitivities' (Figures 4-5)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly makes causal claims ('to isolate the causal effect of each deployment stressor') and uses controlled single-factor ablations (Section 4.3) where all modifiers are disabled except one. This controlled manipulation design is adequate for causal inference about individual stressors."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Robust Real-World Adaptation' and the paper repeatedly frames findings as indicating 'deployment-like robustness' and 'real-world readiness,' but all evidence comes from a synthetic grid game. No explicit bounding of claims to the grid-game setting is provided."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for observed differences between models. Differences could stem from prompt sensitivity, model size, training data composition, or thinking-mode budget differences rather than 'strategy' differences. None of these alternatives are discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures grid game performance (accuracy, score, steps) but frames results as evidence of 'real-world readiness' and 'deployment-like robustness.' The gap between grid game metrics and actual deployment robustness is never acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'GPT-5.2, GPT-5 MINI, GEMINI 3 PRO, GEMINI 3 FLASH, and QWEN3-235B-A22B.' These are marketing names without API versions or snapshot dates. No version identifiers like API dates are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full system and user prompts are provided in Appendix A (Prompts A.1 and A.2). The system prompt describes game mechanics and output format, and the user prompt provides the observation template."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only 'default thinking budget (medium or high)' is mentioned for models supporting thinking mode. Temperature, top-p, max tokens, and other sampling parameters are not reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 describes the player interface in detail: text-only observation, local view with facing direction, state vector, action space, short action history, and event-based execution log. Full prompts are in the appendix."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the game instance generation: 50 random instances per grid size, with parameter ranges specified (noise ∼U(0,0.2), move fail ∼U(0,0.1), latent fraction ∼U(0,0.2)), fixed dynamics (5×5 window, shifts every 25 steps, teleports every 50 steps, drift every 100 steps)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated Limitations or Threats to Validity section exists. The conclusion mentions future work directions but does not discuss limitations of the current study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not state what the results do NOT show or which settings/populations are excluded from claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw trajectory data, episode logs, or per-instance results are made available. Only aggregated results are reported in tables and figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The game instance generation procedure is fully described in Sections 2.1 and 3, including tile placement, parameter sampling ranges, and fixed dynamics schedules."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper evaluates LLM agents on a procedural benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from game generation (random seed → map sampling → parameter assignment) through evaluation (LLM agent plays → metrics computed) is documented in Sections 2 and 3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed anywhere in the paper. Both authors are from Megagon Labs (a corporate research lab) but no funding acknowledgment is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors list Megagon Labs as their affiliation. They are not evaluating a Megagon product—they test third-party models (GPT, Gemini, Qwen)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Megagon Labs (a Recruit Holdings subsidiary) does not appear to have a direct stake in the models evaluated, but without disclosure this cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the five evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether models could have seen similar grid games or the WildGrid code/description during training. While the benchmark is new, the game mechanics could resemble training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Although the benchmark is procedurally generated (making direct contamination unlikely), the paper never explicitly discusses this advantage or the contamination question."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It evaluates LLM agents on a procedural benchmark."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or latency measurements are reported. The study runs 750+ episodes (50 instances × 3 grid sizes × 5 models, each up to 200 steps) but total cost is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or hardware specifications are stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are averaged over 50 random game instances but no seed sensitivity analysis is reported. No variance across instances is shown."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3: 'we generate 50 random game instances for each grid size' for main evaluation and '5 instances for each data point per condition' for ablations."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The choice of parameter ranges (noise, latent fraction, etc.) and thinking budget settings are not justified."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "A single fixed configuration is used for all models with the same parameters. All results for all models are reported—no selection of best configuration occurs."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the benchmark and evaluate third-party models on it. They do not acknowledge the potential bias of benchmark designers selecting game mechanics that may favor certain model capabilities."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of vastly different sizes and compute costs are compared (e.g., Qwen3-235B vs GPT-5 mini) without any discussion of compute budget differences or performance normalized by cost."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper assumes grid game performance reflects 'real-world deployment robustness' but provides no validation of this construct mapping. Whether performance on a grid puzzle predicts robustness in actual deployment scenarios is not examined."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All models use the identical prompt and interaction interface (Section 2.3, Appendix A). The scaffold is controlled across all comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether models' training data could include similar grid game descriptions or solutions, despite the benchmark being novel."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the observation format or prompt structure leaks information that aids performance beyond what a real deployment would provide."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between game instances or whether shared parameter ranges across instances create dependencies."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used or described."
       }
     }
   },
   "claims": [
     {
       "claim": "Performance generally degrades as grid size and horizon increase, with all frontier models exhibiting decreasing accuracy from 6×6 to 10×10.",
       "evidence": "Table 1 shows accuracy declining for GPT-5.2 (48→40→26%), Gemini-3 Pro (50→28→38%), and Gemini-3 Flash (48→42→32%) across grid sizes. Qwen3 fails almost entirely (2→0→0%).",
       "supported": "moderate"
     },
     {
       "claim": "Model rankings are unstable across grid sizes and metrics: weaker models can outperform stronger ones when strategy matches the uncertainty regime.",
       "evidence": "Table 1: Gemini-3 Pro leads on 6×6 and 10×10 accuracy, but Gemini-3 Flash leads on 8×8. GPT-5 mini has lowest steps and best scores despite lower accuracy. Section 4.1 discusses this instability.",
       "supported": "moderate"
     },
     {
       "claim": "Agents trade off completion, efficiency, and penalty avoidance despite no explicit instruction to do so, suggesting partial objective inference.",
       "evidence": "Table 1 shows GPT-5 mini consistently has lowest step counts and best scores despite not leading in accuracy. Section 4.2 shows distinct action-frequency profiles. Section 4.1: 'several models appear to partially infer these objectives.'",
       "supported": "moderate"
     },
     {
       "claim": "LLM agents exhibit a clear sense-then-act signature: information gathering is front-loaded early in episodes.",
       "evidence": "Figure 3 shows SCAN and MEASURE concentrated in earliest steps, decaying toward zero, across all frontier models. GPT-5 MINI shows the strongest early sensing investment.",
       "supported": "moderate"
     },
     {
       "claim": "Single-stressor sensitivities are strongly non-monotonic and model-specific: moderate noise can improve accuracy, and teleportation can aid exploration.",
       "evidence": "Figure 4: GPT-5.2 and Gemini-3 Pro accuracy peaks at mid-range noise. GPT-5.2 achieves near-perfect accuracy with teleports every 10 steps. Section 4.3 discusses these patterns.",
       "supported": "weak"
     },
     {
       "claim": "Robustness in realistic environments depends on adaptive strategy selection as much as raw task-solving capability.",
       "evidence": "Combination of ranking instability (Table 1), strategy-specific strengths (Figures 3-4), and feature attribution showing model-specific sensitivities (Figure 5). This is an interpretive synthesis rather than a directly tested claim.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "50 episodes per grid size for main results and only 5 episodes per data point for ablations, yet no standard deviations, confidence intervals, or error bars are reported anywhere. The observed differences between models could be within noise."
     },
     {
       "flag": "No statistical tests despite comparative claims",
       "detail": "The paper makes many comparative claims (e.g., 'Gemini-3 Pro attains the highest accuracy') by comparing raw numbers without any significance testing. With 50 episodes and high variance in game outcomes, apparent differences may not be statistically significant."
     },
     {
       "flag": "Construct validity gap: grid game vs. real-world robustness",
       "detail": "The paper claims findings about 'real-world readiness' and 'deployment-like robustness' based entirely on a synthetic grid game. No validation that grid game performance predicts actual deployment robustness is provided."
     },
     {
       "flag": "Very small ablation sample size",
       "detail": "Single-stressor ablations use only 5 random episodes per data point (Section 3). The non-monotonic patterns in Figure 4 could easily be noise artifacts at this sample size."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated limitations section or threats to validity discussion, despite significant methodological choices (synthetic setting, small samples, no statistical tests) that warrant explicit acknowledgment."
     },
     {
       "flag": "Overclaiming from synthetic to real-world",
       "detail": "The title promises 'Robust Real-World Adaptation' but all evidence comes from a grid game. The four 'deployment stressors' are parameterized game mechanics, not validated proxies for real deployment conditions."
     }
   ],
   "cited_papers": [
     {
       "title": "Identifying the risks of LM agents with an LM-emulated sandbox",
       "authors": ["Yangjun Ruan", "Honghua Dong", "Andrew Wang", "Silviu Pitis", "Yongchao Zhou", "Jimmy Ba", "Yann Dubois", "Chris J Maddison", "Tatsunori Hashimoto"],
       "year": 2023,
       "arxiv_id": "2309.15817",
       "relevance": "ToolEmu framework for scaling LLM agent safety testing via emulated tools and risk scoring."
     },
     {
       "title": "τ-bench: A benchmark for tool-agent-user interaction in real-world domains",
       "authors": ["Shunyu Yao", "Noah Shinn", "Pedram Razavi", "Karthik Narasimhan"],
       "year": 2024,
       "arxiv_id": "2406.12045",
       "relevance": "Benchmark with persistent state and trajectory-level metrics exposing LLM agent brittleness in multi-step tool use."
     },
     {
       "title": "ToolSandbox: A stateful, conversational, interactive evaluation benchmark for LLM tool use capabilities",
       "authors": ["Jiarui Lu", "Thomas Holleis", "Yizhe Zhang"],
       "year": 2025,
       "relevance": "Evaluates LLM tool use with stateful simulation and conversational interaction, revealing failure modes."
     },
     {
       "title": "Tools Fail: Detecting silent errors in faulty tools",
       "authors": ["Jimin Sun", "So Yeon Min", "Yingshan Chang", "Yonatan Bisk"],
       "year": 2024,
       "arxiv_id": "2406.19228",
       "relevance": "Studies LLM agent robustness to silent tool failures and recovery strategies."
     },
     {
       "title": "Hell or High Water: Evaluating agentic recovery from external failures",
       "authors": ["Andrew Wang", "Sophia Hager", "Adi Asija", "Daniel Khashabi", "Nicholas Andrews"],
       "year": 2025,
       "arxiv_id": "2508.11027",
       "relevance": "Evaluates LLM agent replanning under external failures and ambiguous instructions."
     },
     {
       "title": "OpenAI GPT-5 system card",
       "authors": ["Aaditya Singh"],
       "year": 2025,
       "arxiv_id": "2601.03267",
       "relevance": "System card for GPT-5, one of the models evaluated in this benchmark."
     },
     {
       "title": "CodeAgent: Enhancing code generation with tool-integrated agent systems for real-world repo-level coding challenges",
       "authors": ["Kechi Zhang", "Jia Li", "Ge Li", "Xianjie Shi", "Zhi Jin"],
       "year": 2024,
       "arxiv_id": "2401.07339",
       "relevance": "Tool-integrated LLM agent for code generation, representing the specialized agent paradigm this paper critiques."
     },
     {
       "title": "AlphaEvolve: A coding agent for scientific and algorithmic discovery",
       "authors": ["Alexander Novikov"],
       "year": 2025,
       "arxiv_id": "2506.13131",
       "relevance": "Coding agent for scientific discovery, exemplifying the trend toward specialized LLM agents."
     },
     {
       "title": "BrowserArena: Evaluating LLM agents on real-world web navigation tasks",
       "authors": ["Sagnik Anupam", "Davis Brown", "Shuo Li", "Eric Wong", "Hamed Hassani", "Osbert Bastani"],
       "year": 2025,
       "arxiv_id": "2510.02418",
       "relevance": "Real-world web agent evaluation benchmark where different LLMs adopt distinct behavioral strategies."
     },
     {
       "title": "OdysseyBench: Evaluating LLM agents on long-horizon complex office application workflows",
       "authors": ["Weixuan Wang", "Dongge Han"],
       "year": 2025,
       "arxiv_id": "2508.09124",
       "relevance": "Long-horizon agent evaluation showing performance decline with increasing step count, consistent with this paper's findings."
     },
     {
       "title": "Large language model agent: A survey on methodology, applications and challenges",
       "authors": ["Junyu Luo", "Weizhi Zhang"],
       "year": 2025,
       "arxiv_id": "2503.21460",
       "relevance": "Survey of LLM agent approaches covering planning, tool use, and multi-step reasoning."
     },
     {
       "title": "HAZARD challenge: Embodied decision making in dynamically changing environments",
       "authors": ["Qinhong Zhou", "Sunli Chen", "Yisong Wang"],
       "year": 2024,
       "arxiv_id": "2401.12975",
       "relevance": "Embodied agent benchmark targeting dynamically changing environments, a related robustness evaluation approach."
     }
   ]
 }