ai-research-survey

scan-v5.json (25754B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Learning to Configure Agentic AI Systems",
     "authors": [
       "Aditya Taparia",
       "Som Sagar",
       "Ransalu Senanayake"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.11574",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'up to 25% higher task accuracy' but Table 1 shows improvements of up to 31.3 percentage points over the base model on reasoning tasks; the '25%' figure appears to understate or misrepresent the actual reported numbers without a clear mapping to a specific comparison.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies compare ARC with and without SFT, against flat RL (RL Bandits, RL Episodes), and alternative training objectives (GRPO, DPO), providing reasonable evidence for causal claims about the hierarchical structure's benefit.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion claims ARC is 'a powerful alternative to one-size-fits-all designs' broadly, but results are from 5 benchmarks on 2 models; GAIA performance (6%) is very low and tool-use transfer is explicitly described as 'more weaker,' undermining broad generalization claims.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes GEPA outperforms on MedQA due to domain-specific prompts, but does not systematically consider alternative explanations for ARC's gains (e.g., whether benchmark contamination in backbone LLMs drives results).",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Task accuracy on established benchmarks (GSM8k, DROP, MedQA, HotpotQA, GAIA) is used as the metric, and claims are framed in terms of benchmark accuracy and token cost — measurements and claims are aligned at the same granularity.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has only a brief 'Impact Statement' stating 'We do not foresee any direct societal harm'; there is no dedicated limitations or threats-to-validity section.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed — small GAIA test set (unknown N after 65 training samples), potential benchmark contamination in backbone LLMs, and single-run results without variance reporting are not addressed.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show; the conclusion broadly promotes ARC as 'a practical and effective direction for scaling LLM-based systems' without bounding the claim to the tested benchmarks and model families.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or grant support is mentioned anywhere in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are affiliated with the School of Computing and Augmented Intelligence, Arizona State University, which is clearly disclosed.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are operationally defined: 'workflow' maps to 9 specific patterns (Appendix A), 'configuration' is formally defined as c=(ω,t,b,p) in Section 3, and 'token budget' is given as budget tiers with specific values.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly lists three contributions: the ARC hierarchical RL framework, a hybrid RL+SFT training pipeline, and empirical demonstrations across benchmarks.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 engages substantively with LLM agent frameworks, prompt/workflow optimization (OPRO, DSPy, GEPA, LLMLingua), and hierarchical RL (Options framework, Feudal RL), situating ARC's contributions relative to each thread.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "The abstract says 'Codebase: Github' but no URL is provided; without a verifiable link, this cannot be confirmed as actually released.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "All five benchmarks used (GSM8k, DROP, MedQA, HotpotQA, GAIA) are standard publicly available datasets used unmodified (aside from GAIA partitioning).",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Appendix B mentions 'Python 3.10+' and GPU requirements (40GB VRAM) but provides no requirements.txt, Dockerfile, or exact package versions.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Appendix B describes hardware requirements and Appendix F gives hyperparameters, but without accessible code and without step-by-step instructions, reproduction is not feasible.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Table 1 (main results) reports only point estimates with no error bars or confidence intervals; Table 5 (embedding ablation) includes standard deviations but this is not the primary result table.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are conducted for any comparative claims made in Table 1 or elsewhere.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 reports absolute percentage-point improvements (e.g., '+31.3' for reasoning avg) relative to a base model, providing context for the magnitude of effects.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The number of training episodes (4,000 per dataset minimum) is stated but not justified; GAIA uses only 65 samples for training and an unspecified remainder for evaluation, with no power analysis.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Main results in Table 1 are single-run point estimates with no variance or standard deviation; the paper does not report variance across training runs or seeds.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 includes diverse baselines: base model with tools, grid/greedy search, AutoGen, DSPy, GEPA, LAP, RL Bandits, and RL Episodes.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines include recent systems (AutoGen 2024, DSPy 2023, GEPA 2025) and RL variants; no obviously stale or weak baselines are used.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.5 and Table 4 ablate the SFT refinement stage, compare PPO vs. GRPO, and compare SFT vs. DPO post-training, isolating contributions of each component.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Task accuracy, token cost (Figure 4), workflow diversity entropy and Gini coefficient (Table 2), and reward distribution (Figure 13) are all reported.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "The paper evaluates automated benchmark accuracy; human evaluation is not applicable to this configuration-learning framework.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Standard training/test splits are used for all benchmarks; for GAIA, the first 65 samples are used for training and the remainder for evaluation.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are reported per benchmark and organized by capability axis (reasoning vs. tool-use); Appendix G further breaks down accuracy by workflow type per dataset.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.6 and Appendix I provide detailed error categorization with four failure types and concrete examples for each category.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly reports that GEPA outperforms ARC on MedQA (87.1% vs 64.6%), that GAIA performance is only 6%, and that tool-use cross-task transfer is 'more weaker.'",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Models are named specifically: 'Qwen 2.5 7B Instruct' and 'Gemini 2.5 Flash Lite,' with citations to their technical reports.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions semantic instruction fragments (e.g., 'Decompose the problem', 'Verify intermediate steps') but the full prompt library is not provided; specific system prompts for agents are not shown.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Appendix F provides a comprehensive list of hyperparameters including learning rates, batch size, PPO clip epsilon, discount factor, entropy coefficient, and reward shaping coefficients.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "All 9 agentic workflows are described and illustrated in Appendix A with figures showing LLM call graphs, tool configurations, and inter-agent routing.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "The query feature vector fq is explicitly defined (query length, word count, numerical density, binary indicators for multi-step reasoning and tool use) and the embedding approach using MetaCLIP-H/14 is described.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "All five benchmarks (GSM8k, DROP, MedQA, HotpotQA, GAIA) are publicly available standard datasets; RL episode buffer data is not released but the evaluation data is accessible.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Standard benchmarks are used with their established collection procedures; the GAIA partitioning (first 65 for training, rest for evaluation) is explicitly described.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; all data is from existing public benchmarks.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The data pipeline from query encoding (Section 3.1), through policy decision (Sections 3.1.1–3.1.2), to reward computation (Section 3.2) is fully described with formal notation.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not state the training data cutoffs for Qwen 2.5 7B Instruct or Gemini 2.5 Flash Lite, which is relevant since benchmarks like GSM8k are widely known.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether backbone LLM training data overlaps with benchmark test sets, which is particularly relevant for well-known benchmarks like GSM8k and DROP.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Benchmark contamination is not addressed; GSM8k in particular is a high-risk benchmark for contamination in recent LLMs, and the paper's high accuracy (88.6%) on it is not contextualized relative to this risk.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Figure 4 shows cost per episode in USD using OpenRouter rates for Qwen 2.5 7B Instruct, and the Pareto frontier explicitly compares accuracy vs. cost across all methods.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Appendix B specifies GPU requirements (A100/V100/4090 with 40GB VRAM, or 24GB with 4-bit quantization) and training episodes (4,000 minimum per dataset).",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "ARC achieves up to 25% higher task accuracy compared to fixed architectures while reducing token and runtime costs.",
       "evidence": "Table 1 shows ARC at 72.4%/72.9% avg on reasoning vs. base model at 41.1%/39.4%; Figure 4 shows ARC on Pareto frontier for cost-accuracy tradeoff on GSM8k.",
       "supported": "moderate"
     },
     {
       "claim": "Hierarchical RL (ARC) outperforms flat RL baselines (RL Bandits, RL Episodes).",
       "evidence": "Table 1: ARC 72.4% reasoning avg vs. RL Episodes 65.6% and RL Bandits 63.7% for Qwen 2.5 7B.",
       "supported": "strong"
     },
     {
       "claim": "SFT post-training refinement improves average episode reward by 5–35% across datasets.",
       "evidence": "Section 4.5 and Table 1 compare ARC vs. ARC w/o SFT; accuracy gains of 1–3% are shown, and the 5–35% reward claim is stated in text but not directly tabulated.",
       "supported": "weak"
     },
     {
       "claim": "Policy configuration errors remain below 10% across all benchmarks, meaning most errors stem from inherent LLM limitations.",
       "evidence": "Figure 6 shows policy configuration errors at 8–9% for tool-use tasks and 9% for reasoning tasks.",
       "supported": "strong"
     },
     {
       "claim": "Cross-task policy transfer is governed by shared workflow/tool structure rather than semantic similarity.",
       "evidence": "Table 3 shows HotpotQA→GAIA transfer drops to 2% despite high embedding similarity (Dsim=0.93), while HotpotQA→MedQA transfers well (57%) with lower similarity (0.76).",
       "supported": "moderate"
     },
     {
       "claim": "ARC policies trained on 7B models generalize zero-shot to 32B and 72B variants with monotonically improving performance.",
       "evidence": "Figure 5 shows consistent accuracy gains across Qwen 2.5 7B→32B→72B with Kendall's τ=1.0 and Pearson's r=0.94.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "ARC, a hierarchical RL framework, learns per-query agent configurations (workflow, tools, token budget, prompts) and consistently outperforms fixed architectures, search-based methods, and flat RL baselines on five reasoning and tool-use benchmarks. The two-level policy decomposes the combinatorial configuration space (~62K valid options) into tractable structure and prompt decisions, with a post-RL SFT phase that concentrates the policy on high-reward trajectories. Tool-use benchmark performance remains low (GAIA: 6%), cross-domain transfer depends on structural rather than semantic similarity, and policy configuration errors account for fewer than 10% of failures — the dominant failure modes are LLM reasoning and knowledge gaps.",
   "red_flags": [
     {
       "flag": "No variance on main results",
       "detail": "Table 1 reports single-run point estimates with no standard deviations, confidence intervals, or multiple seeds, making it impossible to assess result reliability."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims (ARC vs. baselines) are made without statistical tests; observed differences could be within noise given single-run reporting."
     },
     {
       "flag": "Code inaccessible",
       "detail": "Abstract says 'Codebase: Github' but no URL is provided; reproducibility cannot be verified."
     },
     {
       "flag": "Very small GAIA evaluation set",
       "detail": "GAIA uses 65 samples for training and an unspecified number for test; all GAIA results (e.g., 6.0%) are based on a handful of examples (≤36 given original GAIA test split size), making them statistically unreliable."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section; only a two-sentence impact statement is provided."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "GSM8k and DROP are high-risk for contamination in recent LLMs; the paper's 88.6% on GSM8k is not contextualized against this risk, and backbone LLM training cutoffs are not stated."
     },
     {
       "flag": "Abstract accuracy claim inconsistent with results",
       "detail": "Abstract claims 'up to 25% higher task accuracy' but Table 1 shows improvements up to 31.3 percentage points over the base model; no clear mapping between the stated '25%' and the reported numbers is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversations",
       "relevance": "Key multi-agent framework baseline that ARC is compared against and builds upon."
     },
     {
       "title": "DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines",
       "relevance": "Prompt/workflow optimization framework used as a baseline; represents the prior state of the art in structured LLM pipeline optimization."
     },
     {
       "title": "GAIA: A Benchmark for General AI Assistants",
       "relevance": "Tool-use benchmark used for evaluation; tests complex multi-modal agentic capabilities."
     },
     {
       "title": "GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning",
       "relevance": "Contemporary competing approach to prompt/workflow optimization; outperforms ARC on MedQA despite ARC's hierarchical structure."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "relevance": "Foundational work on combining reasoning and tool use in LLM agents; directly motivates the agentic workflow design space ARC navigates."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "relevance": "Early work on cost-aware LLM routing; ARC extends this to full agentic workflow configuration."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "relevance": "General agent evaluation suite; related to the benchmark evaluation methodology used in this paper."
     },
     {
       "title": "Training Verifiers to Solve Math Word Problems (GSM8k)",
       "relevance": "Primary reasoning benchmark used for evaluation and training of the ARC policy."
     },
     {
       "title": "SWE-Agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "relevance": "Demonstrates agentic systems in software engineering, part of the broader agent landscape ARC is designed to configure."
     },
     {
       "title": "Large Language Models as Optimizers (OPRO)",
       "relevance": "Prompt optimization baseline; represents the class of prompt-as-optimization-variable approaches that ARC unifies into a hierarchical policy."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly addresses the real practitioner problem of choosing agent configurations from a large combinatorial space, with a working framework applicable to existing LLM backends."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that a lightweight RL policy trained without updating the backbone LLM can navigate a 62K-option configuration space and outperform hand-tuned templates is non-obvious."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or risk concerns are raised; the paper focuses on efficiency and accuracy of agent configurations."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle; straightforward systems paper."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Framework is described as modular and the abstract references a GitHub codebase, suggesting it could be tried with Qwen 2.5 7B on public benchmarks, though no working URL is provided."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Arizona State University, no famous lab or product affiliation."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "47023905",
         "title": "Retrieval-Aware Distillation for Transformer-SSM Hybrids",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=47023905",
         "created_at": "2026-02-15T14:24:02Z"
       }
     ],
     "top_points": 2,
     "total_points": 2,
     "total_comments": 0
   }
 }