ai-research-survey

scan.json (27502B)

---

 {
   "paper": {
     "title": "How Do LLMs Fail In Agentic Scenarios? A Qualitative Analysis",
     "authors": ["JV Roig"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.07497"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository is linked. The agentic evaluation framework (KAMI v0.1) is described but no code URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Section 'Data Availability' states 'The raw experiment data... will be made available at https://docs.kamiwaza.ai/research/datasets.' This is a promise of future release, not current availability."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Execution parameters are listed (temperature 0.4, context window 32K, max tokens 8K, max 20 rounds) but no environment specifications, dependencies, or software versions for the agentic framework are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. Appendices A and B provide test definitions and tool descriptions, but not instructions for running the benchmark."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports 95% t-CI for pooled accuracy of each model (e.g., DeepSeek V3.1: '(91.2%, 93.2%)')."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are performed. Comparisons between models (e.g., Maverick vs. Granite on Q402) rely on raw success counts (2/30 vs 1/30) without any formal test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw success counts (e.g., 13/30, 29/30) and pooled accuracy percentages. No formal effect sizes (Cohen's d, odds ratios) are computed for model comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample of 30 trials per model per scenario (12.5% of 240) is stated but not formally justified. No power analysis or rationale for why 30 is sufficient for the qualitative claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports standard deviation and RSE for each model's pooled accuracy. Table 2 shows run-by-run variation for DeepSeek V3.1 across 8 runs on Q502 and Q602."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three models spanning different performance tiers are compared: Granite 4 Small (58.5%), Llama 4 Maverick (74.6%), DeepSeek V3.1 (92.2%). DeepSeek V3 (59.4%) is also included for architecture-matched comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All models are from 2025: Granite 4 Small (IBM, Oct 2025), Llama 4 Maverick (Meta, Apr 2025), DeepSeek V3.1 (Aug 2025)."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This paper analyzes existing models as-is; there is no system with components to ablate. The V3 vs V3.1 comparison functions as a natural quasi-ablation of post-training."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported as pooled accuracy with CIs (Table 1), per-scenario success rates (e.g., 29/30, 13/30), and qualitative behavioral pattern taxonomies."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The entire study is manual human analysis of 900 execution traces. Section 2.4: 'We performed manual analysis of all 900 sampled execution traces.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "PICARD framework generates fresh randomized data for each trial (randomized file names, text content, CSV data, database records), making each trial inherently novel and preventing memorization."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Detailed per-scenario breakdowns are provided for each model across all 10 scenarios (Q201-Q503), with success/failure counts and pattern taxonomies per scenario."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure case analysis is the core contribution. Section 4 provides extensive trace-level analysis of failure patterns with verbatim execution traces (e.g., malformed JSON, generation loops, schema guessing)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Many negative results are reported: Granite 4 Small failing all 30 Q401 trials, Maverick achieving only 2/30 on Q402, and all models failing on context pollution in Q503."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about scale not predicting agentic robustness are supported by per-scenario data (Maverick 2/30 vs Granite 1/30 on Q402). Four failure archetypes are extensively documented in Section 4. V3 vs V3.1 comparison supports the post-training RL claim."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims DeepSeek V3.1's 'superior reliability derives primarily from post-training reinforcement learning rather than architecture or size.' While the V3/V3.1 comparison (same architecture, different scores) is suggestive, the authors cannot isolate RL from other post-training changes. This is acknowledged only partially in Section 6."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'How Do LLMs Fail In Agentic Scenarios?' is very broad, but results are from only 3 models on 10 scenarios from a single benchmark. Section 6 acknowledges model sample scope and scenario coverage limitations, but the title and abstract frame findings as general LLM phenomena."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: temperature effects, single-tool-per-round constraint influencing Maverick's strategy, scenario coverage bias, and training secrecy preventing attribution. The Chekhov's gun explanation (Section 4.2.2, Q503) discusses Anthropic research on model inclination to use all provided information."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.1 explicitly discusses construct validity — that benchmark scores may not reflect real-world capability. The paper distinguishes between task success rates (proxy) and enterprise reliability (outcome), and Section 5 synthesizes findings into deployment principles that acknowledge this gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with parameter counts are given: 'Granite 4 Small (32B parameters, dense)', 'Llama 4 Maverick (400B total / 17B active, MoE)', 'DeepSeek V3.1 (671B total / 37B active, MoE)'. These are specific open-weight releases with unambiguous identities."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides the complete PICARD test definitions (instruction templates with all randomized fields). Appendix B provides full tool descriptions as loaded into the system prompt. Appendix C shows a complete verbatim conversation sample."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 reports: temperature 0.4, context window 32K tokens for non-thinking models, maximum output tokens 8K per round, maximum 20 inference rounds per trial, single-tool-per-round constraint."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic framework is described: 29 tools across 5 categories (Section 3.2), single-tool-per-round constraint, error feedback mechanism, maximum rounds cap. Tool specifications are fully detailed in Appendix B."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes model selection criteria (3 models from different performance bands), scenario selection (10 of 19, Section 2.2), trial sampling (30 of 240 per scenario, 12.5%), and the emergent coding analysis approach (Section 2.4)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated section discussing five specific validity limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 lists specific threats: only 3 models analyzed, KAMI scenarios may not generalize to long-horizon planning, proprietary post-training prevents behavioral attribution, single-tool-per-round constraint influences Maverick, temperature 0.4 effects unexplored."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states specific scope boundaries: 'Model sample scope. Only three models were analyzed', 'Scenario coverage. KAMI v0.1 tasks emphasize tool-grounded data correctness and may not generalize to long-horizon planning.' Mitigations for v0.2 are stated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw execution traces are promised for future release ('will be made available at https://docs.kamiwaza.ai/research/datasets') but are not currently available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the KAMI v0.1 benchmark process: PICARD framework for randomized trial generation, model execution with specific parameters, trace collection with full conversation histories."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes model selection rationale (different performance bands, architecture diversity). Section 2.2 explains scenario selection criteria (direct agentic capability evaluation). Section 2.3 describes the random sampling of 30 trials per scenario."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: PICARD generates randomized sandbox environments → models execute with 29 tools → traces recorded → 30/240 randomly sampled per model per scenario → manual emergent coding analysis (Section 2.4)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The author is affiliated with Kamiwaza AI, which created and promotes the KAMI benchmark, but no funding statement appears."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation 'Kamiwaza AI' is disclosed in the paper header. Kamiwaza AI is the creator of the KAMI benchmark being promoted."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The author works at Kamiwaza AI, which created the KAMI benchmark. The company has a commercial interest in KAMI being seen as a valuable and insightful benchmark. The paper promotes KAMI's design philosophy and argues for its superiority over traditional benchmarks."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present. The author's commercial interest in the KAMI benchmark through Kamiwaza AI is not explicitly acknowledged as a conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the three models evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The PICARD framework is explicitly designed to prevent contamination through randomized task parameters: 'randomized task parameters (file names, text content, CSV data, and database records) to account for LLM stochasticity and to probe real-world capability rather than memorized responses' (Section 2.3)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.1 extensively discusses benchmark contamination as a motivation. The PICARD framework addresses this by generating fresh randomized data for each trial, making memorization irrelevant. Referenced studies [11,13,15,17,22,26,28-31] document contamination in existing benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyzes LLM execution traces, not human behavior."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API spend, or per-trial compute costs are reported for the 900 analyzed trials."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The referenced KAMI v0.1 paper mentions '5.5 billion tokens' but this paper does not quantify its own compute usage."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "The PICARD framework runs each scenario with randomized parameters multiple times (240 trials per model per scenario). Table 1 reports standard deviation across runs. Table 2 shows 8-run variation for DeepSeek V3.1 on Q502/Q602."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 30 trials sampled per model per scenario, 900 total for qualitative analysis. Full dataset contains ~240 trials per model per scenario (Section 2.3)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature 0.4 was chosen as 'a setting between 0.0 and the typical 0.7 default' (Section 2.3) but no systematic search was conducted. The paper acknowledges 'the effects of alternative temperature settings remain unexplored' (Section 6)."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection or tuning was performed. A single fixed configuration was used for all models."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical hypothesis tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The author created the KAMI benchmark and is evaluating models on it. The bias of evaluating one's own benchmark — including scenario design choices that may favor certain behavioral patterns — is not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses scale (32B vs 400B vs 671B parameters) but does not formally analyze compute budget vs. performance. No compute-matched comparisons are made despite comparing models of vastly different sizes."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.1 extensively discusses construct validity failures in traditional benchmarks. The paper argues KAMI's interactive, multi-step design better measures agentic capability than static Q&A formats. Section 5 discusses why interactive resilience is a better construct for agentic evaluation."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All models use the identical agentic framework with the same 29 tools, same single-tool-per-round constraint, same error feedback mechanism. Section 2.3 specifies consistent execution parameters across all trials."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "PICARD generates fresh randomized data for each trial, making temporal leakage impossible — there are no pre-existing test problems that could appear in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit discussion of whether the evaluation setup leaks information through context (e.g., whether tool descriptions or error messages provide hints beyond what a real deployment would offer)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 30 sampled trials per scenario are independent or share structural similarities through the PICARD randomization scheme."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "PICARD's randomized data generation serves as a concrete contamination prevention method. Section 2.3: 'randomized task parameters... to probe real-world capability rather than memorized responses, following PICARD principles.'"
       }
     }
   },
   "claims": [
     {
       "claim": "Model scale alone does not predict agentic robustness: Llama 4 Maverick (400B) performs only marginally better than Granite 4 Small (32B) on uncertainty-driven tasks.",
       "evidence": "Q402: Maverick 2/30 vs Granite 1/30. Overall KAMI scores: Granite 58.5% vs Maverick 74.6%, but per-scenario gaps are small on hard tasks. Table 1 and Section 4 per-scenario results.",
       "supported": "strong"
     },
     {
       "claim": "DeepSeek V3.1's superior reliability derives primarily from post-training reinforcement learning rather than architecture or size.",
       "evidence": "V3 (59.4%) vs V3.1 (92.2%) with identical architecture (Table 1). However, the authors cannot isolate RL from other post-training changes due to training secrecy (Section 6).",
       "supported": "moderate"
     },
     {
       "claim": "Four recurring failure archetypes cut across model families: premature action without grounding, over-helpfulness, context pollution vulnerability, and fragile execution under load.",
       "evidence": "Documented across all three models with verbatim trace examples in Section 4. Premature schema guessing in Q501-Q503 (all models), over-helpful substitution in Q502 (DeepSeek, Maverick), context pollution in Q503 (all models), generation loops in Q401-Q403 (Granite, Maverick).",
       "supported": "strong"
     },
     {
       "claim": "Recovery capability, not initial correctness, best predicts overall success.",
       "evidence": "DeepSeek V3.1 dominates not by avoiding errors but by consistently recovering. Section 5 synthesis and Table 3 comparing error-recovery traits across models. DeepSeek V3.1 scores 'Very High' on all recovery dimensions.",
       "supported": "moderate"
     },
     {
       "claim": "Explicit prompting interventions significantly improve agentic performance (Q502 vs Q602 comparison).",
       "evidence": "Table 2: DeepSeek V3.1 Q502 avg 52.92% vs Q602 avg 87.50% with only a hint added about missing company data and schema inspection.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["qualitative", "benchmark-eval"],
   "key_findings": "Qualitative analysis of 900 agentic execution traces across three LLMs reveals four recurring failure archetypes: premature action without grounding, over-helpfulness that substitutes missing entities, context pollution vulnerability from distractor data, and fragile execution under cognitive load. Model scale does not predict agentic robustness — Llama 4 Maverick (400B) achieves only 2/30 on multi-CSV analysis, barely above Granite 4 Small's 1/30. DeepSeek V3.1's dominance stems from superior error recovery, likely attributable to post-training reinforcement learning. Simple prompt interventions (e.g., instructing schema inspection) dramatically improve performance (53% → 88% on SQL tasks).",
   "red_flags": [
     {
       "flag": "Author evaluates own benchmark",
       "detail": "JV Roig of Kamiwaza AI created the KAMI benchmark and is evaluating models on it. The paper promotes KAMI's design philosophy and argues for its superiority over traditional benchmarks, without disclosing this as a conflict of interest."
     },
     {
       "flag": "No competing interests statement",
       "detail": "No financial disclosure or competing interests statement despite the author's commercial interest in the KAMI benchmark through Kamiwaza AI."
     },
     {
       "flag": "Small model sample",
       "detail": "Only 3 of ~60 tested models are analyzed qualitatively, yet findings are presented as general LLM failure patterns. The title 'How Do LLMs Fail' overstates the generalizability."
     },
     {
       "flag": "Data not yet released",
       "detail": "The 900 execution traces are promised for future release but not currently available for verification, making the qualitative findings unverifiable."
     },
     {
       "flag": "Single-author qualitative coding",
       "detail": "All 900 traces were manually analyzed by what appears to be a single researcher with no inter-rater reliability assessment. The emergent coding approach (Section 2.4) lacks the multi-coder verification standard in qualitative research."
     }
   ],
   "cited_papers": [
     {
       "title": "Can we trust ai benchmarks? an interdisciplinary review of current issues in ai evaluation",
       "authors": ["Maria Eriksson", "Erasmo Purificato", "Arman Noroozian"],
       "year": 2025,
       "relevance": "Interdisciplinary review of AI benchmark limitations directly relevant to evaluation methodology quality."
     },
     {
       "title": "Leakage in data mining: Formulation, detection, and avoidance",
       "authors": ["Shachar Kaufman", "Saharon Rosset", "Claudia Perlich", "Ori Stitelman"],
       "year": 2012,
       "relevance": "Foundational work on data leakage formalization relevant to benchmark contamination concerns."
     },
     {
       "title": "AI and the everything in the whole wide world benchmark",
       "authors": ["Inioluwa Deborah Raji", "Emily Denton", "Emily M Bender"],
       "year": 2021,
       "relevance": "Critique of AI benchmarks and construct validity failures in ML evaluation."
     },
     {
       "title": "Safetywashing: Do ai safety benchmarks actually measure safety progress?",
       "authors": ["Richard Ren", "Steven Basart", "Adam Khoja"],
       "year": 2024,
       "relevance": "Analysis of whether safety benchmarks have construct validity, directly relevant to benchmark methodology quality."
     },
     {
       "title": "Data contamination through the lens of time",
       "authors": ["Manley Roberts", "Himanshu Thakur", "Christine Herlihy"],
       "year": 2023,
       "relevance": "Studies temporal aspects of benchmark data contamination in LLM evaluation."
     },
     {
       "title": "Benchmark data contamination of large language models: A survey",
       "authors": ["Cheng Xu", "Shuhao Guan", "Derek Greene"],
       "year": 2024,
       "relevance": "Survey of benchmark contamination in LLMs, core concern for evaluation methodology."
     },
     {
       "title": "Benchmarking benchmark leakage in large language models",
       "authors": ["Ruijie Xu", "Zengzhi Wang", "Run-Ze Fan", "Pengfei Liu"],
       "year": 2024,
       "relevance": "Systematic study of benchmark leakage mechanisms in LLM evaluation."
     },
     {
       "title": "Agentic misalignment: How llms could be an insider threat",
       "authors": ["Aengus Lynch", "Benjamin Wright", "Caleb Larson"],
       "year": 2025,
       "relevance": "Anthropic research on agentic LLM failure modes including the 'Chekhov's gun' effect cited in this paper."
     },
     {
       "title": "Testing what models can do, not what they've seen: PICARD: Probing intelligent capabilities via artificial randomized data",
       "authors": ["JV Roig"],
       "year": 2025,
       "relevance": "The PICARD evaluation framework underlying KAMI, proposing randomized data generation to prevent benchmark contamination."
     },
     {
       "title": "Towards a standard, enterprise-relevant agentic ai benchmark: Lessons from 5.5 billion tokens' worth of agentic ai evaluations",
       "authors": ["JV Roig"],
       "year": 2025,
       "relevance": "The KAMI v0.1 benchmark paper describing the full evaluation framework and initial results across ~60 models."
     }
   ]
 }