ai-research-survey

scan.json (26701B)

---

 {
   "paper": {
     "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
     "authors": ["Shunyu Yao", "Noah Shinn", "Pedram Razavi", "Karthik Narasimhan"],
     "year": 2024,
     "venue": "Preprint (under review)",
     "arxiv_id": "2406.12045"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/sierra-research/tau-bench (mentioned in the abstract footnote and in the paper body)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (database JSON files, task instances, domain policies) is released as part of the GitHub repository. The paper states 'We release our codebase publicly to encourage the community to create new tasks and domains for τ-bench.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed dependency listing with versions is mentioned in the paper. The paper mentions Python API tools and various LLM APIs but does not specify environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include step-by-step reproduction instructions. While the code is released, the paper itself does not describe how to run the benchmark experiments, what commands to use, or how to replicate the main results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The main results in Table 2 report only point estimates (e.g., 'gpt-4o: 61.2% retail, 35.2% airline'). No confidence intervals or error bars are reported for these numbers. The pass^k curves in Figure 4 also lack error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'gpt-4o is the best model') but provides no statistical significance tests. All comparisons are based on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute performance levels and differences in context. For example, 'even state-of-the-art LMs like gpt-4o achieve low task success rates (∼61% on τ-retail and ∼35% on τ-airline)' and 'pass^8 drops to < 25%'. The ablation in Table 3 shows specific degradation: 'gpt-4o: 61.2 → 56.8' and '33.2 → 10.8', providing both baseline and changed values."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 115 tasks (τ-retail) and 50 tasks (τ-airline) with at least 3 trials per task, but no justification is given for why these sizes are sufficient for the claims made. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports 'at least 3 trials per task' for main results but does not report standard deviations, variance, or any spread measure across these trials. Only aggregate pass^1 numbers are reported in Table 2."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares multiple models (gpt-4o, gpt-4-turbo, gpt-4-32k, gpt-3.5-turbo, claude-3-opus, claude-3-sonnet, claude-3-haiku, gemini-1.5-pro, gemini-1.5-flash, mistral-large, mixtral-8x22b, meta-llama-3-70B) and multiple methods (Function Calling, ReAct, Act-only) as baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include state-of-the-art models as of mid-2024 (gpt-4o, claude-3-opus, gemini-1.5-pro, meta-llama-3-70B), which were the most advanced models available at the time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes an ablation removing the domain policy from the agent's system prompt (Table 3), showing the impact on performance. It also compares Function Calling vs. ReAct vs. Act-only methods (Figure 3), and reports a 'think' function experiment."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both pass^1 (average success rate) and pass^k (consistency over multiple trials), as well as pass@k for comparison. These capture different aspects of agent performance: average capability vs. reliability."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The authors manually examined 40 failure cases from gpt-4o FC agent trajectories (Section 5.2), classifying them into 4 categories (wrong argument, wrong info, wrong decision, partially resolve). This is manual expert evaluation of system outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no separation of dev and test sets. The same 115 retail and 50 airline tasks are used for all experiments and analysis. The user instructions were iteratively refined using gpt-4-turbo trials (Stage III), creating a risk that the tasks are tuned to specific model behaviors."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides per-domain breakdown (τ-retail vs τ-airline), per-task difficulty visualization (Figure 7), failure category breakdown (Figure 5), and performance by number of write API actions (Figure 6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 provides detailed failure analysis with three failure categories: wrong argument/info (55%), wrong decision (25%), and partial resolution (19%). Specific trajectory examples are shown in Appendix C.2."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that adding a 'think' function for function-calling agents 'did not boost performance' (Section 5.1). It also reports that even the best model (gpt-4o) solves less than 50% of tasks overall, which is itself a negative finding about current agent capabilities."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'even state-of-the-art function calling agents (like gpt-4o) succeed on < 50% of the tasks' (supported by Table 2: avg 48.2%) and 'pass^8 < 25% in retail' (supported by Figure 4). These claims are well-supported by the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (e.g., removing domain policy reduces performance - Table 3, removing reasoning traces reduces performance - ReAct vs Act in Figure 3). These are single-variable controlled ablations, adequate for the claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its claims to the two tested domains (retail and airline customer service) and notes 'For more capable agents in the future, more advanced domains (e.g., medical, tax, or legal) with more complex data and rules can be studied.' The Discussion section acknowledges simulation limitations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Discussion) discusses alternative explanations for results, including user simulator limitations (typos/ambiguities in instructions, limited LM reasoning capacity, potential implicit bias from using gpt-4-turbo for task curation). The paper acknowledges that some failures may be due to user simulation rather than agent issues."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses marketing names without specific version strings for most models: 'gpt-4o', 'gpt-4-turbo', 'claude-3-opus', 'claude-3-sonnet', 'gemini-1.5-pro-latest', 'gemini-1.5-flash-latest'. Only 'gpt-4-32k' is partially versioned as 'gpt-4-32k-0613' in Figure 4, but the main Table 2 uses 'gpt-4-32k'. The user simulator uses 'gpt-4-0613' which is properly versioned."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides complete domain policy documents used as system prompts (Appendix B.1 contains the full retail and airline policies), user instruction templates with concrete examples (Figure 2d, Appendix C.2), and explains that the domain policy is the agent's system prompt. The actual prompts sent to models are fully reconstructable."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states: 'The LM temperature is 0.0 for agent and 1.0 for user.' The paper also specifies 'We limit each task to at most 30 agent actions.' These are the key hyperparameters for the API-based experiments."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the agent scaffolding approaches: Function Calling (FC) where the model autonomously decides to generate a user response or tool call, ReAct with 'Thought: {reasoning} Action: {JSON action}', and Act-only. The user simulation setup with system prompts and conversation history management is also described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents the three-stage benchmark construction process (Section 4): Stage I manual schema/API design, Stage II automatic data generation with LMs (with code shown in Appendix B.2), and Stage III manual task annotation and validation with iterative agent runs. The data generation pipeline is well-documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Discussion) contains a substantive 'Directions for improvement' subsection that discusses specific limitations of the benchmark, including user simulator limitations, task annotation difficulty, and implicit bias from using gpt-4-turbo for task curation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: (1) user instruction typos or ambiguities, (2) user instruction not containing all domain knowledge, (3) user simulation LM capacity limitations at reasoning/calculation/long-context, (4) implicit bias from using gpt-4-turbo FC agent to tune user prompts. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: it focuses on customer service domains (retail and airline) only, uses simplified policies compared to real-world, and notes that results may not extend to more complex domains. The Discussion states 'more advanced domains (e.g., medical, tax, or legal) with more complex data and rules can be studied' in future work."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (database JSON files, task instances, Python API implementations, domain policies) is released through the GitHub repository. This allows independent verification of the benchmark construction and results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the three-stage data collection procedure in detail: manual schema/API design (Stage I), LM-assisted data generation with code (Stage II, with generation code in Appendix B.2), and manual task annotation with iterative validation (Stage III)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The benchmark uses LM-simulated users and programmatic APIs. The data is synthetically generated, not collected from human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The three-stage pipeline is documented with specific details: Stage I manual design, Stage II code-based generation (with actual Python code in Appendix B.2 showing how user profiles are generated), Stage III iterative annotation and validation (with mention of 40+ gpt-4-turbo trials per task). The filtering step of fixing 4 user instruction typos/ambiguities is mentioned."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The Acknowledgements section thanks individuals for feedback and help but does not mention any grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed with their affiliation: Sierra. The first author (Shunyu Yao) has a footnote noting 'Work done during internship.' Author affiliations are clearly stated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The authors are from Sierra, an AI agent company. Sierra has a direct commercial interest in the agent evaluation space. While no explicit funding is disclosed, the work was done at/during an internship at Sierra, which has a financial stake in demonstrating that current agents are not reliable enough (motivating demand for better agent products). The funder independence is not addressed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. Sierra is an AI agent startup, and the authors' equity or other financial interests related to the agent evaluation space are not disclosed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates multiple pre-trained models on the τ-bench benchmark but does not state the training data cutoff dates for any of the models tested. While τ-bench uses synthetic data that is unlikely to be in training sets, the paper does not discuss this."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether any of the benchmark components (domain policies, database schemas, API designs) could have been influenced by or similar to data in the training sets of the tested models. The synthetic data makes contamination less likely but this is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is newly created with synthetic data, making direct contamination unlikely, but the paper does not discuss this. The domain policies and task designs could share patterns with customer service data in training sets, and this is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The benchmark uses LM-simulated users, not real humans."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study. The NeurIPS checklist also marks human subjects items as N/A."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; users are simulated by language models."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; all users are LM-simulated."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; not an experimental study with human subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 'Cost analysis' reports: 'the agent / user simulation costs are $0.38 / $0.23 per task respectively, so running one trial per task costs around 200 dollars.' It also breaks down costs: 'the input prompt / completion output take up 95.9% / 4.1% of the price.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While per-task API costs are reported, the total computational budget for all experiments is not stated. The paper does not mention total API spend across all models and trials, GPU hours, or total compute used."
       }
     }
   },
   "claims": [
     {
       "claim": "Even state-of-the-art function calling agents (like gpt-4o) succeed on less than 50% of τ-bench tasks overall.",
       "evidence": "Table 2 shows gpt-4o achieves 61.2% on τ-retail and 35.2% on τ-airline, with a weighted average of 48.2%. This is the highest among all tested models.",
       "supported": "strong"
     },
     {
       "claim": "Agent consistency drops rapidly with increasing trials: pass^8 < 25% for gpt-4o in retail.",
       "evidence": "Figure 4 shows the pass^k curve for gpt-4o FC in τ-retail, with pass^8 dropping below 25% despite pass^1 being above 60%.",
       "supported": "strong"
     },
     {
       "claim": "Function calling consistently outperforms text-formatted agent methods (ReAct, Act-only) on state-of-the-art models.",
       "evidence": "Figure 3 shows FC outperforming ReAct and Act across gpt-4o, gpt-4-turbo, and gpt-4-32k on τ-retail. However, gpt-3.5-turbo shows a different pattern.",
       "supported": "moderate"
     },
     {
       "claim": "Adding reasoning traces (ReAct vs. Act-only) consistently helps bridge the gap between observations and actions.",
       "evidence": "Figure 3 shows ReAct outperforming Act-only across all four tested models on τ-retail.",
       "supported": "moderate"
     },
     {
       "claim": "Removing domain policy from the agent significantly hurts gpt-4o on τ-airline (−22.4%) but only slightly on τ-retail (−4.4%).",
       "evidence": "Table 3 shows performance degradation with policy removal: gpt-4o retail 61.2→56.8, airline 33.2→10.8. gpt-3.5-turbo shows minimal degradation on airline (10.8→9.6).",
       "supported": "strong"
     },
     {
       "claim": "Current agents struggle most with complex database reasoning (~55% of failures) and domain rule following (~25% of failures).",
       "evidence": "Section 5.2 and Figure 5 provide a breakdown of 36 failed gpt-4o FC trajectories in τ-retail: wrong argument (19.4%), wrong info (25.0%), wrong decision (22.2%), partially resolve (33.3%). Wrong argument + wrong info = ~44%, and wrong decision = ~22%.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "τ-bench introduces a benchmark for evaluating language agents on realistic tool-agent-user interaction with domain-specific policies. Even the best model (gpt-4o with function calling) achieves only 48.2% average task success, and consistency drops sharply with repeated trials (pass^8 < 25% on retail). The main failure modes are complex database reasoning (wrong arguments/info, ~55% of failures), domain rule following (wrong decisions, ~25%), and compound request handling (~19%). Removing domain policies shows that agents rely heavily on commonsense in simpler domains but benefit significantly from policy documents in complex domains.",
   "red_flags": [
     {
       "flag": "Task design bias from model-in-the-loop annotation",
       "detail": "Task instances were iteratively refined using gpt-4-turbo FC agent runs (Stage III), creating potential bias where tasks are tuned to the behavioral patterns of one specific model. The paper acknowledges this: 'there is also some element of implicit bias during the task curation process since we use the gpt-4-turbo FC agent to tune the user's system prompt.'"
     },
     {
       "flag": "No confidence intervals on main results",
       "detail": "All results in Table 2 are point estimates without error bars or confidence intervals, despite running only 3+ trials per task. With 115 tasks and 3 trials each, the estimates have non-trivial sampling uncertainty that is not quantified."
     },
     {
       "flag": "Company affiliation not discussed as conflict",
       "detail": "All authors are from Sierra, an AI agent company. The finding that 'current agents are not reliable enough for real-world deployment' directly motivates demand for Sierra's products. This conflict of interest is not acknowledged in the paper."
     },
     {
       "flag": "No held-out test set",
       "detail": "All 165 tasks are used for both development (iterative refinement with gpt-4-turbo) and evaluation. There is no held-out split to guard against overfitting the benchmark to specific model behaviors."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Karthik Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLM agents on real-world software engineering tasks, directly relevant to the survey's scope of agent evaluation methodology."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["Xiao Liu"],
       "year": 2023,
       "arxiv_id": "2308.03688",
       "relevance": "Benchmark for evaluating LLMs as agents across multiple environments, directly comparable to τ-bench as an agent evaluation methodology."
     },
     {
       "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents",
       "authors": ["Shuyan Zhou"],
       "year": 2023,
       "arxiv_id": "2307.13854",
       "relevance": "Realistic web-based agent benchmark that τ-bench extends by adding human-in-the-loop interaction, relevant to agent evaluation methodology quality."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik Narasimhan", "Yuan Cao"],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational agent framework used as a baseline method in τ-bench, key to understanding agent scaffolding approaches."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "relevance": "Introduced pass@k metric for code generation, which τ-bench extends to pass^k for agent reliability evaluation."
     },
     {
       "title": "Reflexion: An Autonomous Agent with Dynamic Memory and Self-Reflection",
       "authors": ["Noah Shinn", "Beck Labash", "Ashwin Gopinath"],
       "year": 2023,
       "relevance": "Agent self-reflection method discussed in context of why it's unsuitable for real-world user-facing settings, relevant to agent methodology evaluation."
     },
     {
       "title": "Identifying the Risks of LM Agents with an LM-Emulated Sandbox",
       "authors": ["Yangjun Ruan"],
       "year": 2023,
       "arxiv_id": "2309.15817",
       "relevance": "ToolEmu benchmark that uses LMs to emulate tool execution for safety evaluation, relevant to agent safety and evaluation methodology."
     },
     {
       "title": "Berkeley Function Calling Leaderboard",
       "authors": ["Fanjia Yan"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating LLM function calling capabilities, directly relevant to tool-use evaluation methodology."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation Framework",
       "authors": ["Qingyun Wu"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Multi-agent conversation framework relevant to agentic AI architectures and evaluation."
     },
     {
       "title": "Generative Agents: Interactive Simulacra of Human Behavior",
       "authors": ["Joon Sung Park"],
       "year": 2023,
       "arxiv_id": "2304.03442",
       "relevance": "Foundational work on LM-based simulation of human behavior, directly relevant to τ-bench's user simulation approach."
     },
     {
       "title": "Cognitive Architectures for Language Agents",
       "authors": ["Theodore R. Sumers", "Shunyu Yao", "Karthik Narasimhan", "Thomas L. Griffiths"],
       "year": 2023,
       "arxiv_id": "2309.02427",
       "relevance": "Survey of cognitive architectures for language agents, providing theoretical grounding for agent evaluation."
     },
     {
       "title": "Toolformer: Language Models Can Teach Themselves to Use Tools",
       "authors": ["Timo Schick"],
       "year": 2023,
       "arxiv_id": "2302.04761",
       "relevance": "Foundational work on LLM tool use, relevant to understanding the tool-use capabilities evaluated by τ-bench."
     }
   ]
 }