ai-research-survey

scan.json (29249B)

---

 {
   "paper": {
     "title": "Learning to Configure Agentic AI Systems",
     "authors": [
       "Aditya Taparia",
       "Som Sagar",
       "Ransalu Senanayake"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.11574"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ARC, a hierarchical RL framework that dynamically selects workflows, tools, budgets, and prompts per query, outperforms fixed architectures and flat RL baselines across five benchmarks. On GSM8k, ARC achieves 88.6% accuracy vs 83.6% for GEPA and 85.2% for flat RL. ARC occupies Pareto-optimal accuracy-cost regions, demonstrating that per-query adaptive configuration reduces computational waste. Policies trained on 7B models transfer zero-shot to 32B and 72B variants, though cross-task transfer is limited by tool-structure overlap.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Codebase: Github' which appears to be a hyperlinked repository reference in the PDF."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All five benchmarks (GSM8k, DROP, MedQA, HotpotQA, GAIA) are publicly available standard datasets. The paper did not collect proprietary data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B describes high-level requirements ('Python 3.10+ installation and a CUDA-capable machine', GPU class) but provides no requirements.txt, Dockerfile, or specific library versions needed to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "Algorithm 1 and Appendix F provide detailed hyperparameters and training procedure, but there are no step-by-step reproduction instructions (commands to run, scripts to execute) in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 (main results) reports only point estimates for all methods across all benchmarks. No confidence intervals, error bars, or ± notation on evaluation results. The embedding ablation (Table 5) has ± but that is a separate component evaluation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. All claims of 'outperforms' are based on comparing raw accuracy numbers without any p-values, t-tests, or other statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports both absolute accuracy values and delta improvements (e.g., '+31.3', '+33.5') against the base model, providing sufficient context to understand the magnitude of improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for benchmark sizes or evaluation set sizes. For GAIA, only the first 65 samples are used for training with the rest for evaluation, but no justification for this split. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main evaluation results (Table 1) report single numbers with no variance, standard deviation, or spread measures across runs. Figure 9 shows training dynamics variance but not evaluation variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines across four categories: base models with tools, search-based methods (Grid/Greedy Search), optimization frameworks (AutoGen, DSPy, GEPA, LAP), and flat RL baselines (RL Bandits, RL Episodes)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent frameworks: AutoGen (2024), DSPy (2023), GEPA (2025). The models evaluated (Qwen 2.5, Gemini 2.5) are current."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablation studies: SFT refinement (Table 1, ARC vs ARC w/o SFT), alternative training objectives (Table 4, PPO vs GRPO, SFT vs DPO), embedding models (Appendix D, 19 models evaluated), prompt generators (Appendix E, 12 models), and action masking (Figure 3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Beyond task accuracy, the paper evaluates cost efficiency (Figure 4, token-based API cost per episode), workflow diversity (Table 2, entropy and Gini coefficient), and transfer performance (Table 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs. All evaluation is automated using ground-truth answer matching on benchmarks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'We employ standard training split for policy learning and evaluate on the test (or validation) set.' GAIA: 'the first 65 samples for training and the rest for evaluation.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by individual benchmark (Table 1), by reasoning vs tool-use capability axes, by workflow type (Figure 12), and errors by category (Figure 6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6 provides error analysis with four error types and distribution across benchmarks (Figure 6). Appendix I includes specific failure case examples for each error category."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "GEPA outperforms ARC on MedQA (87.1% vs 64.6%), which the paper discusses. GRPO and DPO underperform PPO/SFT (Table 4). Transfer to GAIA from HotpotQA fails (2.0% vs 6.0% in-domain). LAP baseline shows no meaningful improvement."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'up to 25% higher task accuracy' but the conclusion states 'up to 16% higher accuracy.' This internal inconsistency on the key magnitude claim undermines trust. The largest gain vs a strong baseline in Table 1 is ARC (63.9%) vs GEPA (39.3%) on DROP = 24.6pp, but most gains are 5-14pp."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies provide controlled single-variable manipulations: removing SFT (Table 1), replacing PPO with GRPO/DPO (Table 4), comparing hierarchical vs flat RL. The causal claims about component contributions are supported by adequate ablation design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Learning to Configure Agentic AI Systems' and the 'Model Agnostic' claim (Section 4.2) suggest broad generalizability, but experiments cover only 5 benchmarks spanning reasoning and simple QA, with 2 model families. The paper does not test on agentic coding, web navigation, or other agent tasks. Transfer analysis (Section 4.4) partially bounds claims but the title overreaches."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the improvements. Could the gains stem from increased total compute exploration during RL training? Could the prompt library quality (GPT-5.2 generated) be the dominant factor rather than the hierarchical structure? The MedQA discussion (Section 4.2) touches on prompt content but no systematic consideration of confounds."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures benchmark accuracy and token/dollar cost — claims match the granularity of measurements. It does not overclaim that benchmark accuracy represents broader 'agent capability' or 'intelligence.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Main evaluation models specified as 'Qwen 2.5 7B Instruct' (Yang et al., 2024a) and 'Gemini 2.5 Flash Lite' (Comanici et al., 2025), with paper references. These identify specific model releases (family, size, variant). Scaling experiments use Qwen 2.5 32B and 72B."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompts are described as 'semantic instruction fragments' like 'Decompose the problem' and 'Verify intermediate steps' (Section 3.1.2), but the actual prompt text used in experiments is not provided. The prompt library was generated by GPT-5.2 via meta-prompting but the full prompts are not in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Extensive hyperparameters in Appendix F: learning rates (3e-4 structure, 5e-5 prompt), batch size (32), PPO clip epsilon (0.2), discount factor (0.95), entropy coefficient (0.05), all reward shaping coefficients (α=5.0, βs=0.02, βt=0.03, δ1=0.1, δ2=0.2, δ3=0.3), SFT parameters, and GRPO/DPO hyperparameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "All 9 workflow patterns are described in detail in Appendix A with diagrams (Figure 7), including LLM call counts, agent roles, and tool configurations per workflow. The hierarchical policy architecture (Section 3.1), action masking (Section 3.2), and tool allocation are thoroughly described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "State representation construction is documented: MetaCLIP-H/14 embeddings concatenated with hand-crafted features (query length, word count, numerical density, binary indicators). Standard benchmark splits used, with GAIA partitioning explicitly described (first 65 for training, rest for evaluation)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. Section 6 'Impact Statement' is brief and only addresses societal harm. Limitations are scattered throughout results discussion but not consolidated."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats-to-validity discussion. The paper discusses MedQA underperformance and transfer limitations as results observations, not as systematic threats to the study's validity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do NOT show. The paper does not bound its claims to the tested benchmarks, models, or task types. No mention of settings where ARC would not apply."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-episode rewards, individual predictions, RL training logs) is released. Only aggregated results in tables and figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The RL training data collection is described: episodes are generated by executing configured agent systems on benchmark queries, producing correctness signals and cost statistics. Benchmarks are standard public datasets with sources cited."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (GSM8k, DROP, MedQA, HotpotQA, GAIA)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 documents the full pipeline: query encoding → structure policy sampling → action masking → prompt composition → workflow execution → reward computation → buffer storage → PPO update → elite filtering → SFT refinement."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed anywhere in the paper. No acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly stated: all three authors are from School of Computing and Augmented Intelligence, Arizona State University. No industry affiliations to disclose."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding disclosure is itself a reporting gap."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosures are present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of training data cutoff dates for Qwen 2.5 7B Instruct or Gemini 2.5 Flash Lite, despite evaluating these models on public benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GSM8k (2021), DROP (2019), MedQA (2021), or HotpotQA (2018) questions appeared in the training data of Qwen 2.5 or Gemini 2.5."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All five benchmarks were published years before the models' likely training cutoffs. GSM8k, DROP, MedQA, and HotpotQA are widely known and likely in training data. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated benchmark testing."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows accuracy vs cost trade-off with 'token-based API cost per episode using OpenRouter rates for Qwen2.5 7B Instruct.' Cost efficiency is a central contribution of the paper."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B describes hardware requirements and Appendix F lists '4,000 per dataset minimum' training episodes, but total GPU hours, training wall-clock time, or total API spend are not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Table 1 are point estimates with no indication of multiple random seed evaluation. No seed sensitivity analysis is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation runs producing the Table 1 results is not stated. It is unclear whether results are from a single run or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Appendix F lists final hyperparameter values but does not report how they were selected or how many configurations were tried. No search method or budget is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how final hyperparameters were selected. No mention of validation-based selection or systematic search procedure."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons across 5 benchmarks and 10+ baselines with no statistical tests at all, let alone correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement RL Bandits, RL Episodes, and LAP baselines. No acknowledgment that author-implemented baselines may systematically underperform vs the authors' own method."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 presents a Pareto frontier of accuracy vs cost ($ per episode) on GSM8K, showing ARC achieves better accuracy-cost trade-offs. This explicitly plots performance as a function of compute cost."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GSM8k accuracy, DROP accuracy, etc. actually measure the 'agentic configuration' capability the paper claims. The paper uses these benchmarks without questioning their validity for evaluating dynamic agent configuration."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Scaffold differences are the explicit research variable. The paper compares different scaffold configurations (9 workflows) and demonstrates that scaffold choice significantly affects results (Figure 12). The comparison against baselines with different scaffolds is acknowledged as comparing configuration strategies, not attributing differences to models alone."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. GSM8k (2021), DROP (2019), MedQA (2021), and HotpotQA (2018) were all published years before Qwen 2.5 and Gemini 2.5 training cutoffs. Solutions may be in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks answer information through context or other channels."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and test examples share structural similarities or come from overlapping distributions."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No canary strings, membership inference tests, decontamination pipelines, or temporal splits."
       }
     }
   },
   "claims": [
     {
       "claim": "ARC achieves up to 25% higher task accuracy over strong baselines while reducing token and runtime costs.",
       "evidence": "Table 1 shows ARC (88.6% GSM8k, 63.9% DROP, 64.6% MedQA) vs various baselines. Largest improvement over a strong baseline: ARC (63.9%) vs GEPA (39.3%) on DROP = 24.6pp. Figure 4 shows Pareto-optimal cost-accuracy trade-off.",
       "supported": "moderate"
     },
     {
       "claim": "Hierarchical RL (ARC) consistently outperforms flat RL baselines (RL Bandits, RL Episodes) across all benchmarks.",
       "evidence": "Table 1: ARC vs RL Episodes on GSM8k (88.6% vs 85.2%), DROP (63.9% vs 54.3%), MedQA (64.6% vs 57.3%), HotpotQA (34.1% vs 27.8%), GAIA (6.0% vs 2.0%).",
       "supported": "strong"
     },
     {
       "claim": "SFT refinement consistently improves performance by 5-35% reward improvement across all datasets and models.",
       "evidence": "Table 1 shows ARC vs ARC w/o SFT: GSM8k 88.6% vs 87.6% (+1.0%), DROP 63.9% vs 62.3% (+1.6%), MedQA 64.6% vs 58.4% (+6.2%), HotpotQA 34.1% vs 33.7% (+0.4%), GAIA 6.0% vs 5.0% (+1.0%). Accuracy gains are 0.4-6.2%, not 5-35%.",
       "supported": "weak"
     },
     {
       "claim": "Policies trained on 7B models transfer zero-shot to 32B and 72B model variants with monotonic performance improvement.",
       "evidence": "Figure 5 shows performance improving with scale across 4 benchmarks. 'Avg. Kendall's τ = 1.0 and Pearson's r = 0.94' (Section 4.4).",
       "supported": "moderate"
     },
     {
       "claim": "Policy configuration errors remain below 10% across all benchmarks.",
       "evidence": "Figure 6 shows policy configuration errors at 8% (GAIA), 9% (HotpotQA, MedQA) across all benchmarks. Error categorization methodology in Appendix I.",
       "supported": "moderate"
     },
     {
       "claim": "ARC is model-agnostic, showing consistent gains across both open-weight (Qwen 2.5) and proprietary (Gemini 2.5) models.",
       "evidence": "Table 1 shows ARC improvements on both Qwen 2.5 7B (+31.3 avg reasoning, +13.95 avg tool-use) and Gemini 2.5 FL (+33.5 avg reasoning, +8.95 avg tool-use).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Abstract-conclusion inconsistency on key claim",
       "detail": "The abstract claims 'up to 25% higher task accuracy' while the conclusion states 'up to 16% higher accuracy.' This inconsistency on the paper's headline number suggests imprecise or cherry-picked reporting."
     },
     {
       "flag": "No error bars or statistical tests on main results",
       "detail": "Table 1 reports single-run accuracy numbers for all methods across all benchmarks with no variance, confidence intervals, or significance tests. Given that RL training is stochastic, the absence of multi-seed evaluation is a significant omission."
     },
     {
       "flag": "No contamination analysis despite using old benchmarks with new models",
       "detail": "GSM8k (2021), DROP (2019), MedQA (2021), and HotpotQA (2018) are all old enough to be in the training data of Qwen 2.5 and Gemini 2.5. Base model accuracy improvements from configuration could partially reflect better retrieval of memorized solutions rather than genuine reasoning improvement."
     },
     {
       "flag": "SFT improvement claim overstated",
       "detail": "The paper claims SFT 'improves average episode reward by ≈5–35% across all datasets' but Table 1 shows accuracy improvements of only 0.4-6.2pp. The 5-35% figure refers to reward (which includes cost penalties) not accuracy, but this distinction is not made clear."
     },
     {
       "flag": "Very low GAIA scores make improvements hard to interpret",
       "detail": "GAIA scores range from 0-6% across all methods. ARC's 6.0% vs 2.0% base model (4pp improvement) is reported as meaningful, but with such low absolute scores on what appears to be a small evaluation set, these differences may not be statistically significant."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated discussion of limitations, threats to validity, or scope boundaries. The Impact Statement (Section 6) addresses only societal harm ('We do not foresee any direct societal harm') without methodological self-critique."
     }
   ],
   "cited_papers": [
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik R. Narasimhan", "Yuan Cao"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational work on LLM agents combining reasoning and tool use, directly relevant to the agentic workflow patterns ARC configures."
     },
     {
       "title": "Toolformer: Language Models Can Teach Themselves to Use Tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì"],
       "year": 2023,
       "relevance": "Key work on LLM tool use that ARC builds upon, demonstrating autonomous tool selection by language models."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversations",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2024,
       "relevance": "Multi-agent framework used as a baseline; represents fixed workflow/prompt optimization approaches ARC claims to improve upon."
     },
     {
       "title": "DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines",
       "authors": ["Omar Khattab", "Arnav Singhvi", "Paridhi Maheshwari"],
       "year": 2023,
       "arxiv_id": "2310.03714",
       "relevance": "Prompt optimization framework used as a baseline, representing the prompt-as-optimization-variable approach to LLM configuration."
     },
     {
       "title": "GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning",
       "authors": ["Lakshya A. Agrawal", "Sherry Tan", "Deniz Soylu"],
       "year": 2025,
       "arxiv_id": "2507.19457",
       "relevance": "Prompt evolution framework that outperforms ARC on MedQA (87.1% vs 64.6%), demonstrating domain-specific prompt content can dominate structural optimization."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Agentic software engineering system representing the class of agent architectures ARC aims to dynamically configure."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Cost-aware LLM routing framework addressing the efficiency dimension ARC optimizes, evolving from model selection to full configuration."
     },
     {
       "title": "GAIA: A Benchmark for General AI Assistants",
       "authors": ["Grégoire Mialon", "Clémentine Fourrier", "Thomas Wolf", "Yann LeCun", "Thomas Scialom"],
       "year": 2023,
       "relevance": "Multi-modal tool-use benchmark used for evaluation; challenging benchmark where all methods score below 6%."
     },
     {
       "title": "Lost in the Middle: How Language Models Use Long Contexts",
       "authors": ["Nelson F. Liu", "Kevin Lin", "John Hewitt"],
       "year": 2024,
       "relevance": "Demonstrates the lost-in-the-middle phenomenon that motivates ARC's context budget optimization."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["Xiao Liu", "Hao Yu", "Hanchen Zhang"],
       "year": 2023,
       "arxiv_id": "2308.03688",
       "relevance": "Agent evaluation suite relevant to understanding how agentic systems perform across diverse tasks."
     },
     {
       "title": "Token-Budget-Aware LLM Reasoning",
       "authors": ["Tingxu Han", "Zhenting Wang", "Chunrong Fang"],
       "year": 2025,
       "relevance": "Token budget management for LLM reasoning, addressing the same compute-efficiency dimension ARC optimizes with learned policies."
     },
     {
       "title": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-World APIs",
       "authors": ["Yujia Qin", "Shihao Liang", "Yining Ye"],
       "year": 2023,
       "arxiv_id": "2307.16789",
       "relevance": "Large-scale tool-use framework expanding the action space available to LLM agents."
     }
   ]
 }