ai-research-survey

scan.json (24280B)

---

 {
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "paper": {
     "title": "SABER: Small Actions, Big Errors — Safeguarding Mutating Steps in LLM Agents",
     "authors": ["Alejandro Cuadron", "Pengfei Yu", "Yang Liu", "Arpit Gupta"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.07850",
     "doi": "10.48550/arXiv.2512.07850"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "An anonymous repository is provided: https://anonymous.4open.science/r/SABER-1E54/ (Appendix B, Reproducibility Statement)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The τ-Bench Verified corrected datasets are included in the repository per Appendix B. They also use the public SWE-Bench Verified and τ-Bench benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, or detailed environment specifications in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B states the repository contains 'experiment drivers, prompts, and evaluation pipelines for all models' and 'scripts to regenerate figures and tables.'"
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports average scores over three runs but no confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports p-values from logistic regression for mutating vs non-mutating distance effects (all p < 0.001 for mutating, varying for non-mutating)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports odds ratios (OR) with coefficients, e.g., OR=0.04 for Claude Sonnet 4 on Airline mutating distance. Percentage point improvements are also reported throughout Section 6.1."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Sample sizes are reported (n=297, n=345, n=690 in Table 1) but no justification for why these sizes are adequate or any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 states results are 'average score over three runs' to 'reduce variance' but no standard deviation or spread measure is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Each model is compared with and without SABER (Table 2), providing clear baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-5, Claude Sonnet 4, and Qwen3-Thinking-235B — all contemporary state-of-the-art models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides ablation removing reflection, verification, and their combination on τ-Bench Verified."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only task success rate is reported as a metric across all benchmarks. No secondary metrics (e.g., efficiency, number of turns, latency)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation uses automated metrics (task success on benchmarks with simulated users). No human evaluation of system outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "τ-Bench and SWE-Bench Verified are established held-out test sets. The paper also creates τ-Bench Verified as a corrected version."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by benchmark domain (Airline vs Retail vs SWE-Bench) and by original vs Verified variants (Table 2)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 provides detailed examples of failure cases in τ-Bench (Figures 1 and 3). The mutating vs non-mutating analysis itself is a form of failure analysis."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings: SABER slightly hurts Claude Sonnet 4 on τ-Bench-V Retail (82.3%→81.0%), and SWE-Bench gains are modest since only reflection can be applied. The Qwen3-Thinking self-pairing underperforms the Instruct auxiliary."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of +28% relative on Airline, +11% on Retail, +7% on SWE-Bench for Qwen3-Thinking are supported by Table 2 data (49.3→63.3 is ~28% relative on Airline)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about mutating actions driving failure are supported by logistic regression with controlled variables (Table 1). Ablation study (Table 3) supports causal claims about SABER components through controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim broadly about 'LLM Agents' but results are on only three benchmarks (τ-Bench Airline/Retail and SWE-Bench Verified). The paper argues for 'action-level analysis' and 'targeted safeguards' as general 'prerequisites for robust multi-turn agents' without bounding to tested domains."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations. For example, the mutating/non-mutating distinction might correlate with task difficulty or action frequency rather than being causal. The ceiling effect on Retail (Table 3) is noted but not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures task success rate on benchmarks with simulated users and frames this as evidence about 'robust multi-turn agents' in 'real-world' settings without discussing the gap between simulated and actual user interactions."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model versions specified: 'claude-sonnet-4-20250514', 'gpt-5-2025-08-07', 'Qwen3-235B-A22B-Thinking-2507', 'Qwen3-235B-A22B-Instruct-2507' (Section 6.1)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the reflection and verification prompts conceptually (Section 4) but does not provide actual prompt text. The repository may contain them, but the paper itself does not."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or other sampling hyperparameters reported. Only the block cap N=16 for context cleaning is stated (Table 3 caption)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "SABER's scaffolding is described in detail in Section 4: mutation-gated verification, targeted reflection injection, block-based context filtering with embedding similarity retrieval. Figure 2 provides a workflow diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 and Appendices C-D document the τ-Bench Verified corrections in detail, with specific task-by-task fixes. The data pipeline from original τ-Bench to Verified is fully documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Limitations' provides substantive discussion of two key limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses specific threats: SABER is external rather than internalized, and dependence on user/user-simulator for verification is a specific limitation of the approach."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section discusses design limitations but does not explicitly state what the results do NOT show (e.g., does not bound claims to the specific benchmark domains tested or acknowledge that results may not transfer to other agent architectures)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B states the repository contains data and scripts to reproduce all results, including execution traces and corrected datasets."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Execution traces are collected from running models on τ-Bench and SWE-Bench Verified. The benchmarks, models, and evaluation protocols are described in Section 6.1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data comes from benchmark evaluations with simulated users."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from trajectory collection → mutating/non-mutating classification → logistic regression is described in Section 3. The τ-Bench corrections are documented task-by-task in Appendices C-D."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement. Authors are from Amazon AGI Foundations but no explicit funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are listed as affiliated with 'Amazon AGI Foundations' on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Amazon has commercial interest in agent reliability (Amazon Q Developer is mentioned in related work). The funder/employer has a stake in demonstrating agent improvements."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the models used (GPT-5, Claude Sonnet 4, Qwen3)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether τ-Bench or SWE-Bench tasks may have appeared in training data of the evaluated models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "τ-Bench was published in 2024 and SWE-Bench in 2024 — models trained after these dates may have seen them. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; all evaluation uses simulated users and automated benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency reported despite SABER adding an auxiliary model for every mutating step. The overhead is described qualitatively ('marginal') but not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget stated. Multiple runs across multiple models and benchmarks with no cost reporting."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are averaged over 3 runs (Table 2 caption) but no per-seed results or sensitivity analysis is provided."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 caption: 'we report the average score over three runs.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the block cap N=16 or other SABER parameters were selected, or what search budget was used."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The N=16 block cap and auxiliary model pairing choices appear tuned but selection rationale is not provided."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons across models, benchmarks, and configurations in Table 1 with no correction for family-wise error rate."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors evaluate their own SABER system against baselines without acknowledging self-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "SABER adds an auxiliary model call for every turn (classification + potential reflection/verification). This compute overhead is not quantified or compared to baseline compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 extensively discusses construct validity issues in τ-Bench, identifying annotation errors and underspecification that cap performance. This is a central contribution of the paper."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly evaluates the scaffold (SABER) as the intervention being tested, comparing same models with and without SABER. The auxiliary model pairing is also varied (Table 3, Section 6.1)."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether models were trained on τ-Bench or SWE-Bench data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information not available in real deployment."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between training data and benchmark problems."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Deviations in mutating actions reduce odds of success by 55-96% per deviation, while non-mutating deviations have little effect (<10% per deviation).",
       "evidence": "Table 1: logistic regression across 3 models on τ-Bench Airline/Retail with p-values. OR ranges from 0.04-0.45 for mutating, 0.81-0.99 for non-mutating.",
       "supported": "strong"
     },
     {
       "claim": "SABER improves Qwen3-Thinking by +19.7pp on τ-Bench-V Airline (58.5%→78.2%) and +10.8pp on Retail (66.9%→77.7%).",
       "evidence": "Table 2, averaged over 3 runs.",
       "supported": "moderate"
     },
     {
       "claim": "SABER improves Qwen3-Thinking by ~4pp on SWE-Bench Verified (42.6%→45.1%).",
       "evidence": "Table 2, SWE-Bench Verified row. Only reflection can be applied on SWE-Bench.",
       "supported": "moderate"
     },
     {
       "claim": "τ-Bench contains annotation errors and underspecified instructions that cap model performance (Airline ~70%, Retail ~92%).",
       "evidence": "Section 5 with detailed corrections in Appendices C-D documenting 24 Airline and 18 Retail task corrections.",
       "supported": "strong"
     },
     {
       "claim": "Mutating actions account for only 14-18% of total steps but dominate failure risk.",
       "evidence": "Section 6.1 reports percentages (e.g., Qwen3-Airline: 15.5%, Claude-4-Retail: 18.1%).",
       "supported": "strong"
     },
     {
       "claim": "Reflection and verification are synergistic: each adds ~10pp on Airline, together yield 78.7%.",
       "evidence": "Table 3 ablation study on Qwen3-Thinking for τ-Bench Verified.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Mutating (state-changing) actions in LLM agent trajectories account for only 14-18% of steps but are the dominant driver of task failure, with each mutating deviation reducing success odds by 55-96%. SABER, a model-agnostic safeguard combining mutation-gated user verification, targeted reflection, and context cleaning, delivers consistent improvements across models and benchmarks (up to +19.7pp). The paper also identifies systematic annotation errors in the widely-used τ-Bench benchmark and releases a corrected τ-Bench Verified version.",
   "red_flags": [
     {
       "flag": "Company evaluating agent reliability improvements",
       "detail": "All authors are from Amazon AGI Foundations. Amazon sells agent-based developer tools (Amazon Q Developer, referenced in related work). The paper's findings about improving agent reliability serve Amazon's commercial interests, yet no conflict of interest is disclosed."
     },
     {
       "flag": "No variance reporting despite multi-run averaging",
       "detail": "Table 2 reports averages over 3 runs but provides no standard deviation, confidence intervals, or range. With only 3 runs, some observed improvements (e.g., Claude Sonnet 4 on Retail: 73.3%→78.3%) may not be statistically significant."
     },
     {
       "flag": "No cost analysis for added auxiliary model",
       "detail": "SABER requires an auxiliary model call on every turn for mutation classification, plus reflection and verification for mutating steps. This doubles or triples inference cost but no cost analysis is provided, making practical applicability unclear."
     },
     {
       "flag": "Simulated users as proxy for real human verification",
       "detail": "SABER's mutation-gated verification relies on user confirmation, but evaluation uses Claude Sonnet 4 as simulated user. The gap between simulated and real user behavior is acknowledged in limitations but not quantified."
     }
   ],
   "cited_papers": [
     {
       "title": "τ-bench: A benchmark for tool-agent-user interaction in real-world domains",
       "authors": ["Shunyu Yao", "Noah Shinn", "Pedram Razavi", "Karthik Narasimhan"],
       "year": 2024,
       "arxiv_id": "2406.12045",
       "relevance": "Primary benchmark used for evaluation; paper identifies and corrects annotation errors in it."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Key benchmark for evaluating coding agent capabilities on real-world software engineering tasks."
     },
     {
       "title": "AI agents that matter",
       "authors": ["Sayash Kapoor", "Benedikt Stroebl", "Zachary S. Siegel"],
       "year": 2024,
       "arxiv_id": "2407.01502",
       "relevance": "Discusses evaluation pitfalls for AI agents including construct validity and meaningful measurement."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri", "Melissa Z. Pan", "Shuyi Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analyzes failure modes in multi-agent LLM systems, complementary to SABER's single-agent failure analysis."
     },
     {
       "title": "Which agent causes task failures and when? On automated failure attribution of LLM multi-agent systems",
       "authors": ["Shaokun Zhang", "Ming Yin", "Jieyu Zhang"],
       "year": 2025,
       "arxiv_id": "2505.00212",
       "relevance": "Failure attribution in multi-agent systems — directly related to understanding where agent errors occur."
     },
     {
       "title": "OpenHands: An open platform for AI software developers as generalist agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2025,
       "arxiv_id": "2407.16741",
       "relevance": "Agent framework used for SWE-Bench evaluation in this paper."
     },
     {
       "title": "Lost in the middle: How language models use long contexts",
       "authors": ["Nelson F. Liu", "Kevin Lin", "John Hewitt"],
       "year": 2023,
       "arxiv_id": "2307.03172",
       "relevance": "Key finding on context utilization that motivates SABER's targeted reflection and context cleaning."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Influential agent framework for software engineering tasks."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational reasoning+acting framework that SABER builds upon for reflection injection."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong", "Mingchen Zhuge", "Jiaqi Chen"],
       "year": 2024,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent framework; provides context for SABER's single-agent approach comparison."
     },
     {
       "title": "Qwen3 technical report",
       "authors": ["An Yang", "Anfeng Li", "Baosong Yang"],
       "year": 2025,
       "arxiv_id": "2505.09388",
       "relevance": "Technical report for one of the primary models evaluated in the paper."
     }
   ]
 }