ai-research-survey

scan.json (29566B)

---

 {
   "paper": {
     "title": "In-Context Distillation with Self-Consistency Cascades: A Simple, Training-Free Way to Reduce LLM Agent Costs",
     "authors": [
       "Vishnu Sarukkai",
       "Asanshay Gupta",
       "James Hong",
       "Michaël Gharbi",
       "Kayvon Fatahalian"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.02543",
     "doi": "10.48550/arXiv.2512.02543"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "In-context distillation—retrieving teacher demonstrations and inserting them into a cheaper student model's prompt—combined with self-consistency cascades achieves 2.5× cost reduction on ALFWorld at teacher-level accuracy (96% vs 89%) and 2× cost reduction on AppWorld at iso-accuracy. The method requires no fine-tuning or weight updates and generalizes to open-weight models (Llama-3.3-70B). Breakeven versus always using the teacher occurs after only 843 episodes on ALFWorld. All results are single-run point estimates without error bars or variance reporting.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL, GitHub link, or archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: ALFWorld (Shridhar et al., 2020) and AppWorld (Trivedi et al., 2024). No proprietary data was collected beyond teacher demonstrations, which are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions MiniLM-L6-v2 for embeddings and API access to OpenAI/Anthropic, but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The method is described algorithmically (Algorithms 1-2) but without runnable reproduction guidance."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as single point estimates (e.g., '0.87 accuracy', '0.96 accuracy'). No confidence intervals or error bars appear in any table or figure."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims its method 'dominates alternatives' and 'establishes a new Pareto frontier' based solely on comparing raw numbers. No statistical significance tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout: '2.5× cost reduction... reducing per-episode costs from $0.059 to $0.024' (Section 6.1), accuracy improvements from 0.18 to 0.87 (Section 6.1), and detailed Tables 1, 4, 5 provide absolute and relative comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Test sets are 134 tasks (ALFWorld) and 168 tasks (AppWorld), determined by the benchmarks. No power analysis or justification for whether these sizes are sufficient for the comparative claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. All results appear to be single-run. There is no mention of multiple experimental runs or seed variation."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines included: Teacher (Claude Sonnet 4.5), Student (ZS), Student (IC), Student (Cascade only), Random Mix at various teacher fractions (0.2-0.8), and GPT-4.1 zero-shot. See Section 5 and Figure 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines use current models: Claude Sonnet 4.5, GPT-4.1-mini, GPT-4.1, Llama-3.3-70B. They also compare against CuGA (Marreed et al., 2025), a concurrent compound system."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Systematic ablations test each component: IC alone vs Cascade alone vs IC+Cascade (Figure 2), varying k (number of examples, Figure 3), varying DB size (Figure 4), per-step vs single retrieval (Table 3, Appendix A), and varying temperature for cascades."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two primary metrics: task accuracy (episode success rate) and cost (normalized to teacher baseline and absolute USD). Both are reported throughout."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of agent outputs or trajectories. All evaluation is automated via benchmark task success criteria (unit tests for AppWorld, goal completion for ALFWorld)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Clear train/test separation: ALFWorld uses train split (500 demos) and evaluates on 'eval-out-of-distribution' (134 tasks). AppWorld uses train+val (147 tasks) for demos and evaluates on 'test-normal' (168 tasks)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-difficulty breakdown on AppWorld (difficulty 1/2/3). Results are also broken down by benchmark, model (GPT-4.1-mini vs Llama-3.3-70B), and retrieval configuration."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 discusses when the method fails: task divergence from demos, ambiguous states, contradictory retrievals. Table 2 shows significant accuracy degradation on difficulty-3 tasks (43% vs teacher's 71%). Section 6.2 reports fine-tuning failure on AppWorld."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Fine-tuning GPT-4o-mini on AppWorld 'failed when training on our teacher demonstrations' and Llama-3.3-70B 'trained but produced 0% task success despite converging on training loss' (Section 6.2). AppWorld accuracy fluctuates with k values (Section 6.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 2.5× cost reduction on ALFWorld (from $0.059 to $0.024), matching teacher accuracy, and 2× cost reduction on AppWorld at iso-accuracy are all supported by Section 6.1, Tables 4-5, and Figure 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about which components drive performance ('in-context distillation bridges the teacher-student gap', 'combined system dominates alternatives') are supported by controlled ablations: IC alone, Cascade alone, and IC+Cascade are tested systematically with single-variable differences."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'A Simple, Training-Free Way to Reduce LLM Agent Costs' broadly, but results are on only two benchmarks (ALFWorld and AppWorld). The abstract claims the approach 'makes advanced agentic systems economically viable for a broader range of applications' without bounding this generalization to tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper offers one speculative explanation for why the student exceeds the teacher on ALFWorld ('likely because retrieved examples provide... implicit information about environment dynamics') but does not systematically discuss alternative explanations for the observed results, confounds, or robustness to other factors."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures task success rate and inference cost, and frames results in terms of task success rate and inference cost. No proxy gap exists—claims match the granularity of measurements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers are provided: 'Claude Sonnet 4.5' (teacher), 'GPT-4.1-mini' (student), 'GPT-4.1' (comparison), 'Llama-3.3-70B' (open-weight student), 'MiniLM-L6-v2' (embeddings). These are distinct API model names sufficient for reproduction."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides full prompt templates for Plan, ReAct, and Verifier agents, including system_prompt and user_prompt with exact text. The templates show how retrieved examples are inserted."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 reports: temperature=0.1 (default), max 4096 output tokens, N=3 for self-consistency sampling, k=6 (ALFWorld) and k=3 (AppWorld) for retrieval. Various temperatures (0.1, 0.5, 1.0) tested for cascades."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The ReAct-style agent architecture is described in detail (Section 4.1, Algorithm 1), retrieval mechanism with multi-key embeddings (Section 4.3), self-consistency cascade logic (Section 4.4, Algorithm 2), and the complete pipeline (Figure 1)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 documents demonstration collection: teacher trajectories include observations, reasoning traces, and actions. Dense embeddings computed with MiniLM-L6-v2 for goal, plan, and per-step reasoning. Benchmarks are used as-is with specified train/test splits."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations section. Section 7 (Discussion) positions the work positively without substantively discussing limitations. No threats to validity are addressed."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The Discussion section focuses on positioning the method's advantages rather than examining potential weaknesses or threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It acknowledges fine-tuning may be better for 'stable, high-volume production systems' (Section 7) but does not bound claims to the tested benchmarks, models, or task types."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (teacher trajectories, per-episode results, retrieval logs) is released. Only aggregate results are shown in tables and figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes teacher demonstration collection: the teacher model is run on Tdemo tasks to produce complete trajectories with goals, plans, observations, reasoning traces, and actions. Dense embeddings are computed using MiniLM-L6-v2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (ALFWorld, AppWorld)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: demonstration collection (Section 4.2), embedding and indexing, per-step retrieval (Section 4.3), self-consistency cascade (Section 4.4). Token counts and cost calculations are detailed in Appendix C."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements state: 'Support for this project was provided by Reve and Roblox, and API credits were provided by OpenAI and together.ai.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Stanford University (Sarukkai, Gupta, Fatahalian) and Reve (Hong, Gharbi)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI provided API credits and GPT-4.1-mini (an OpenAI product) is the primary student model shown to perform well. OpenAI has a commercial interest in demonstrating cost-effective use of their cheaper models. Reve co-authored the paper, meaning the funder is directly involved in research design and outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Authors from Reve (a company) do not declare whether they have equity or other financial interests related to the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (Claude Sonnet 4.5, GPT-4.1-mini, GPT-4.1, Llama-3.3-70B)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ALFWorld (2020) or AppWorld (2024) tasks could appear in model training data. ALFWorld has been public for 5 years and is widely referenced."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ALFWorld was published in 2020 and is a well-known benchmark likely present in training data of recent models. AppWorld is from 2024. Neither contamination risk is discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed inference costs reported: absolute USD per episode ($0.059 teacher, $0.024 IC+Cascade on ALFWorld; $0.589 vs $0.17 on AppWorld), normalized costs, and full token-level breakdowns in Appendix C (Tables 4-5)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Teacher demonstration costs quantified ($29.50 for 500 ALFWorld demos, $86.73 for 147 AppWorld demos). Per-episode costs detailed with per-model API pricing (Appendix C.1). Breakeven analysis provided (Appendix C.5)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity. All results appear to be single-run. Given the stochastic nature of LLM sampling (even at temperature 0.1), seed variation could affect results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It appears all results are from single runs. No 'averaged over K runs' statements appear anywhere."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "While Figures 3-4 sweep k (number of examples) and DB size, no systematic hyperparameter search budget is reported. The search over temperatures and k values is shown but the total compute spent on tuning is not stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The selection of k=6 (ALFWorld) and k=3 (AppWorld) is justified via the cost-accuracy analysis in Section 6.3 and Figure 3, showing diminishing returns. The rationale for the chosen operating points is clear."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many pairwise comparisons are made across methods, benchmarks, and configurations without any statistical tests, let alone correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors implement all methods (baselines, ablations, their own) without acknowledging potential bias from evaluating their own system against their own baseline implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 explicitly plots cost vs performance for all methods. The entire paper is organized around cost-accuracy tradeoffs, with normalized cost on the x-axis and accuracy on the y-axis."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 briefly justifies benchmark selection ('tasks that share structural patterns... enabling cross-task learning') but does not discuss whether ALFWorld and AppWorld actually measure the real-world agent cost-reduction capabilities claimed. No discussion of construct validity or comparison with alternative benchmarks."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All model comparisons use the same ReAct-style scaffolding framework (Algorithm 1). The scaffold is held constant across teacher, student, and ablation configurations, isolating the model and distillation effects."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ALFWorld was published in 2020. Models trained in 2024-2025 could have seen its tasks. No temporal leakage analysis is discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The retrieved teacher demonstrations themselves are a form of information injection, but the paper does not discuss whether benchmark-specific information leaks through retrieval."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses benchmark-provided train/test splits but does not verify independence or discuss potential structural similarities between demonstration tasks and test tasks beyond their different splits."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied (no canary strings, membership inference, decontamination, or temporal analysis)."
       }
     }
   },
   "claims": [
     {
       "claim": "In-context distillation with self-consistency cascades achieves 2.5× cost reduction on ALFWorld at teacher-level accuracy, reducing per-episode costs from $0.059 to $0.024.",
       "evidence": "Figure 2 and Table 4 show IC+Cascade achieves 96% accuracy at 0.42 normalized cost ($0.024/episode) vs teacher at 89% accuracy ($0.059/episode).",
       "supported": "strong"
     },
     {
       "claim": "The combined system exceeds teacher accuracy on ALFWorld (96% vs 89%).",
       "evidence": "Table 1 and Section 6.1 report 0.96 vs 0.89 accuracy. However, this is a single-run result on 134 tasks with no variance reporting, so the difference may not be statistically significant.",
       "supported": "moderate"
     },
     {
       "claim": "The method achieves 2× cost reduction on AppWorld at iso-accuracy.",
       "evidence": "Section 6.1 and Figure 2 show IC+Cascade achieves 0.66 accuracy at 0.29 normalized cost, compared to Random Mix achieving similar accuracy at higher cost.",
       "supported": "strong"
     },
     {
       "claim": "The approach generalizes to open-weight LLMs (Llama-3.3-70B).",
       "evidence": "Table 1 shows Llama-3.3-70B follows similar trends: ALFWorld 0.50→0.87→0.93 (ZS→IC→IC+Cascade), AppWorld 0.11→0.32→0.44.",
       "supported": "moderate"
     },
     {
       "claim": "Breakeven versus teacher-only occurs after 843 episodes on ALFWorld and 207 on AppWorld.",
       "evidence": "Appendix C.5 provides the calculation: N* = $29.50 / ($0.059 - $0.024) ≈ 843 for ALFWorld; N* = $86.73 / $0.42 ≈ 207 for AppWorld.",
       "supported": "strong"
     },
     {
       "claim": "The method is competitive with CuGA, a specialized compound agentic system, achieving 65.5% vs 73.2% on AppWorld.",
       "evidence": "Section 6.2 compares IC+Cascade (65.5%) against CuGA's leaderboard result (73.2%), noting CuGA requires more engineering investment. With oracle routing, their method reaches 77.6%.",
       "supported": "weak"
     },
     {
       "claim": "In-context distillation is a training-free alternative to fine-tuning.",
       "evidence": "Section 6.2 reports fine-tuning GPT-4o-mini matched their accuracy on ALFWorld (94% vs 96%), but fine-tuning failed entirely on AppWorld (API failures, 0% success despite training loss convergence).",
       "supported": "moderate"
     },
     {
       "claim": "Per-step retrieval reduces cost without sacrificing accuracy compared to single retrieval.",
       "evidence": "Table 3 (Appendix A) shows per-step retrieval matches single-retrieval accuracy (0.87 vs 0.87 ALFWorld, 0.55 vs 0.54 AppWorld) at lower cost (0.43 vs 0.54, 0.15 vs 0.24).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance reporting",
       "detail": "All results are single-run point estimates on relatively small test sets (134 and 168 tasks). The claim that the student exceeds teacher accuracy (96% vs 89% on 134 tasks) could easily fall within random variation. Without variance across seeds or confidence intervals, the statistical reliability of all comparisons is unknown."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations section. The Discussion (Section 7) frames the work positively without acknowledging weaknesses such as the narrow benchmark selection, single-run results, or potential contamination concerns."
     },
     {
       "flag": "Potential funder conflict",
       "detail": "OpenAI provided API credits and GPT-4.1-mini (OpenAI's product) is the primary student model shown to perform well. Reve is both a funder and has co-authors on the paper. These relationships are disclosed but their potential influence on study design is not discussed."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "ALFWorld has been public since 2020 and is widely used in LLM agent research. Models trained in 2024-2025 may have seen its tasks. The paper does not discuss this, yet reports that the student exceeds teacher accuracy—which could be partly explained by training data contamination."
     },
     {
       "flag": "Broad claims from narrow evaluation",
       "detail": "The title promises 'A Simple, Training-Free Way to Reduce LLM Agent Costs' broadly, but results are on only two benchmarks (one embodied planning, one API orchestration). No real-world deployment or diverse task domains are tested."
     }
   ],
   "cited_papers": [
     {
       "title": "Self-generated in-context examples improve LLM agents for sequential decision-making tasks",
       "authors": ["Vishnu Sarukkai", "Zhiqiang Xie", "Kayvon Fatahalian"],
       "year": 2025,
       "relevance": "Prior work by same authors on retrieval-augmented agent improvement; directly foundational to the in-context distillation approach."
     },
     {
       "title": "Large language model cascades with mixture of thoughts",
       "authors": ["Murong Yue", "Jie Zhao"],
       "year": 2024,
       "relevance": "Proposes self-consistency-based cascades for LLM routing, which is the core deferral mechanism adapted in this paper."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Foundational work on LLM cost reduction through model cascades and routing strategies."
     },
     {
       "title": "RouteLLM: Learning to route LLMs from preference data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu"],
       "year": 2025,
       "relevance": "Learning-based LLM routing from preference data; represents the trained-router alternative to self-consistency cascades."
     },
     {
       "title": "Language model cascades: Token-level uncertainty and beyond",
       "authors": ["Neha Gupta", "Harikrishna Narasimhan", "Wittawat Jitkrittum"],
       "year": 2024,
       "relevance": "Token-level cascade methods for cost-accuracy tradeoffs in LLM inference."
     },
     {
       "title": "Distilling step-by-step! Outperforming larger language models with less training data and smaller model sizes",
       "authors": ["Cheng-Yu Hsieh", "Chun-Liang Li", "Chih-Kuan Yeh"],
       "year": 2023,
       "arxiv_id": "2305.02301",
       "relevance": "Reasoning distillation approach showing smaller models can match larger ones; foundational to the in-context distillation concept."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2022,
       "arxiv_id": "2203.11171",
       "relevance": "Proposes self-consistency for improving LLM reasoning; the uncertainty signal mechanism adopted for cascade routing."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2023,
       "relevance": "ReAct framework used as the base agent architecture in this paper's experiments."
     },
     {
       "title": "AppWorld: A controllable world of apps and people for benchmarking interactive coding agents",
       "authors": ["Harsh Trivedi", "Tushar Khot", "Mareike Hartmann"],
       "year": 2024,
       "relevance": "One of the two primary evaluation benchmarks; tests multi-step API workflow agents."
     },
     {
       "title": "Fireact: Toward language agent fine-tuning",
       "authors": ["Baian Chen", "Chang Shu", "Ehsan Shareghi"],
       "year": 2023,
       "arxiv_id": "2310.05915",
       "relevance": "Agent fine-tuning approach representing the parametric distillation alternative that this paper aims to avoid."
     },
     {
       "title": "AgentTuning: Enabling generalized agent abilities for LLMs",
       "authors": ["Aohan Zeng", "Mingdao Liu", "Rui Lu"],
       "year": 2023,
       "arxiv_id": "2310.12823",
       "relevance": "Training-based approach to generalizing agent capabilities; contrasts with training-free in-context distillation."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath"],
       "year": 2023,
       "relevance": "Self-improving agent using verbal feedback; related approach to agent capability improvement without weight updates."
     }
   ]
 }