ai-research-survey

scan.json (26917B)

---

 {
   "paper": {
     "title": "SAGE: Steerable Agentic Data Generation for Deep Search with Execution Feedback",
     "authors": ["Fangyuan Xu", "Rujun Han", "Yanfei Chen", "Zifeng Wang", "I-Hung Hsu", "Jun Yan", "Vishy Tirumalashetty", "Eunsol Choi", "Tomas Pfister", "Chen-Yu Lee"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.18202",
     "doi": "10.48550/arXiv.2601.18202"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SAGE proposes a dual-agent pipeline for generating difficulty-controlled deep search QA pairs, using a data generator and a search agent with iterative execution feedback. The framework achieves 87% correctness and 50% pass rate (correct + target difficulty) after 3 feedback rounds, significantly outperforming resampling baselines. Training Qwen-7B on SAGE-generated data yields up to 29% relative improvement on in-domain evaluation and 23% on out-of-domain FRAMES, and enables transfer from Wikipedia-based retrieval to Google Search at inference time.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'The code and data will be released at https://github.com/carriex/sage' — this is a promise of future release, not an actual release."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Same future-release promise. While evaluation uses public benchmarks (Musique, FRAMES, NQ, HotpotQA), the SAGE-generated training data (20K examples) is not yet released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section A.3 mentions '8 H100 GPUs' and some training parameters, but no requirements.txt, Dockerfile, or library version specifications are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions. The paper provides algorithmic descriptions (Algorithms 1-2) and prompts (Figures 4-8), but no runnable scripts or reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-5 are point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., '27% relative improvement') based solely on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Relative improvements are reported with baseline context: '27% relative improvement' (28.5% vs 22.4%), '23% relative improvement' on FRAMES (32.3% vs 26.2%). Tables show absolute values enabling magnitude assessment."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Training set size of 20K is matched to baselines for fair comparison, but no justification for why 300 test samples per split is sufficient. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures reported across runs. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines: NQ+HotpotQA (Search-R1 checkpoints), Musique, prompting-based Gemini agents, and data generator without difficulty control. Both intrinsic and extrinsic evaluations include baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Search-R1 (Jin et al., 2025) is the most recent RL-based search agent framework. Gemini-2.5-flash is contemporary. Baselines are current and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 ablates resampling vs. feedback across rounds. Table 5 ablates number of feedback rounds (0-3) for downstream performance. Figure 2 breaks down by target step."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Intrinsic evaluation uses four metrics: % correct, % pass, Avg@4, and # search steps (Table 2). Downstream evaluation uses accuracy via LLM-as-judge on multiple benchmarks."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated using LLM-as-judge (gemini-2.0-flash). No human evaluation of generated data quality or agent outputs is conducted."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "In-domain test data (300 per step) is separately generated from training data. Out-of-domain evaluation uses separate benchmarks (Musique test split, FRAMES, GAIA, Browsecomp, HLE-Search)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides per-hop breakdowns (3-hop through 7-hop). Figure 2 breaks down by target step. Table 4 reports per-benchmark. Figure 3 analyzes reasoning type distribution."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 and Table 6 provide detailed error categorization: superficial complexity (13%), multi-query collapse (21%), overly specific questions (31%), information co-location (35%), and four incorrect-data categories."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 shows that 3 rounds of feedback does not improve over 2 rounds despite generating harder data, leading to the observation that 'increasing data difficulty alone is insufficient.' QWEN-3B achieves 1.0% on Browsecomp across all training data."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims 'up to 23% relative performance gain' on out-of-domain benchmarks — Table 3 shows QWEN-7B FRAMES: 32.3% vs 26.2% (23.3% relative). '27% relative improvements on in-domain' — 28.5% vs 22.4% for 3B. Claims match reported numbers."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims (e.g., 'execution feedback improves data quality') are supported by controlled ablations in Table 2 comparing resampling vs. feedback holding other variables constant, and Table 5 varying only the number of feedback rounds."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Limitations section explicitly bounds claims: 'only experiment with generating data from a single general domain corpus, Wikipedia,' 'do not explore alternative reinforcement learning algorithms such as GRPO, nor model scales beyond 7B parameters.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why SAGE-trained agents outperform baselines. Could the improvement be due to data diversity rather than difficulty? Or the filtering criteria? These confounds are not addressed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's intrinsic metrics (correctness, search steps) directly measure what they claim. For downstream evaluation, they measure accuracy on QA benchmarks and frame results as QA performance, matching the granularity of measurement."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Uses 'gemini-2.5-flash' and 'gemini-2.0-flash' without snapshot dates or API versions. 'Qwen-2.5-3B-Instruct' and 'Qwen-2.5-7B-Instruct' are specific model names but Gemini versions are marketing names only."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Figures 4, 5, 6, 7, 8, and 10 in the appendix, covering the data generator, search agent, LLM judge, and analysis prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section A.2 reports temperature=1, thinking disabled for Gemini. Section A.3 reports learning rate 1e-6/1e-5, 500 steps, batch size 512, mini-batch 256, micro-batch 64, GAE λ=1, γ=1, max sequence 8192 tokens."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The ReACT-based search agent framework is described (Section 2.1). The dual-agent pipeline is detailed with Algorithms 1-2 showing data generator and search agent interaction, feedback loops, and stopping criteria."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data filtering is documented: 'filter out questions that require less than two search steps,' pass@K=0 questions are removed (Section 3.3). Training data size is 20K. E5 retriever returns top 3 passages per query."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Dedicated 'Limitations' section with two subsections: 'Proposed method' and 'Experimental setting,' providing substantive discussion of the framework's constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats identified: fixed search agent may limit data quality, pass@K=1 may admit incorrect content, single-corpus limitation, no exploration of GRPO, and model scale limited to 7B."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries: 'only focuses on generating high-quality (q,a) pairs for RL training,' 'only experiment with generating data from a single general domain corpus, Wikipedia,' 'do not explore... nor model scales beyond 7B.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Generated data is not yet released. Only aggregated results are reported. No way to independently verify the generated QA pairs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data generation procedure is described in detail: Algorithm 1 outlines the full pipeline from document sampling to QA pair generation and verification. Corpus is the 2018 Wikipedia dump with E5 retriever."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is synthetically generated from a standard public corpus (Wikipedia). Standard benchmarks used for evaluation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: random document sampling → initial QA generation with difficulty prompt → search agent verification (K=4 traces) → execution feedback → filtering (pass@K, min steps). Algorithms 1-2 formalize this."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding acknowledgment. The paper notes 'Work done at Google Cloud AI Research' and 'This work has no implications of any Google products' but does not disclose funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly stated: Google Cloud AI Research (8 authors) and New York University (2 authors, one shared). First author's internship at Google is noted."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Most authors are Google employees. The pipeline uses Google's Gemini models as the data generator and search agent. Google has a commercial interest in demonstrating the effectiveness of Gemini for data generation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for Qwen-2.5 or Gemini models. The 2018 Wikipedia dump is specified for retrieval but pre-training cutoffs are not discussed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Qwen or Gemini pre-training data overlaps with Musique, FRAMES, GAIA, or other evaluation benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HotpotQA (2018), Musique (2022), and other benchmarks were published well before likely Qwen/Gemini training cutoffs. No contamination analysis performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, latency, or cost-per-example reported for the data generation pipeline or inference. The pipeline makes extensive API calls to Gemini (generator + K search traces × R rounds per example) without quantifying cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training hardware mentioned ('single node with 8 H100 GPUs') but total GPU hours, wall-clock time, and API spend for data generation are not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity for either data generation or model training."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "K=4 traces for search agent verification and K=8 for Table 1 difficulty estimation are stated. However, the number of training runs for downstream evaluation is not stated — results appear to be single-run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. Training uses specific hyperparameters (Section A.3) without stating how they were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports downstream performance for 0-3 feedback rounds and selects 2 rounds based on best downstream performance. All configurations are shown transparently."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons across models, datasets, and settings but no statistical tests are performed, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors evaluate their own data generation pipeline against baselines without acknowledging evaluation bias. Search-R1 baselines use public checkpoints, but Musique-trained models are their own reimplementation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "SAGE data generation requires multiple Gemini API calls per example (generator + K×R search traces), making it far more expensive than static datasets. This compute cost difference is not discussed relative to performance gains."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmarks (Musique, FRAMES, GAIA, Browsecomp, HLE-Search) actually measure 'deep search' capability or what construct validity gaps might exist."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All model comparisons use the same training framework (Search-R1/PPO) and same retrieval setup (E5 + Wikipedia 2018). The scaffold is held constant, isolating the training data as the variable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Qwen models may have been trained on data including Musique/HotpotQA solutions. The 2018 Wikipedia dump predates the models but benchmark answers could be in pre-training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. LLM-as-judge uses gemini-2.0-flash which may have internalized benchmark answers."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether SAGE-generated training data and in-domain test data are sufficiently independent, despite both being generated from the same corpus with the same pipeline."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods applied."
       }
     }
   },
   "claims": [
     {
       "claim": "SAGE generates data that is both more correct (87%) and more difficult than baselines after 3 feedback rounds, with 50% pass rate vs 47% for resampling.",
       "evidence": "Table 2 shows progressive improvement: initial generator 71% correct/18% pass → +3 feedback: 87% correct/50% pass vs +3 resample: 84% correct/47% pass.",
       "supported": "strong"
     },
     {
       "claim": "Training on SAGE data yields 27% relative improvement on in-domain evaluation for QWEN-3B (28.5% vs 22.4% Musique) and 29% for QWEN-7B (38.1% vs 29.6%).",
       "evidence": "Table 3 reports per-step and average accuracy for models trained on different data. Improvements consistent across both model sizes.",
       "supported": "moderate"
     },
     {
       "claim": "Up to 23% relative improvement on out-of-domain FRAMES benchmark for QWEN-7B (32.3% vs 26.2% when trained on NQ+HotpotQA).",
       "evidence": "Table 3, FRAMES column. However, these are single-run results with no error bars or significance tests, so true improvement magnitude is uncertain.",
       "supported": "moderate"
     },
     {
       "claim": "Agents trained with fixed-corpus retrieval can transfer to Google Search at inference time, showing 50% relative improvement on GAIA for QWEN-7B (24.0% vs 15.6%).",
       "evidence": "Table 4 shows improvements on GAIA but mixed results on other benchmarks: marginal on Browsecomp (2.6% vs 2.1%) and no gain on HLE-Search (7.0% vs 8.0%).",
       "supported": "moderate"
     },
     {
       "claim": "SAGE generates questions requiring more diverse reasoning strategies than Musique, including calculation (35% vs 5%) and temporal reasoning (32% vs 8%).",
       "evidence": "Figure 3 shows reasoning type distribution based on LLM analysis of 100 trajectories each. However, the analysis uses gemini-2.5-flash to label reasoning types, not human annotation.",
       "supported": "moderate"
     },
     {
       "claim": "Increasing feedback beyond 2 rounds does not improve downstream performance despite generating harder data, suggesting difficulty alone is insufficient.",
       "evidence": "Table 5: round 2 achieves 38.1% in-domain, 32.3% FRAMES; round 3 drops to 34.1% and 28.1% respectively, despite Avg@4 decreasing from 80.4 to 79.5.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "8 of 10 authors are from Google Cloud AI Research. The pipeline relies on Google's Gemini models for data generation, search agent, and LLM-as-judge evaluation. While the paper states 'This work has no implications of any Google products,' the demonstrations showcase Gemini capabilities."
     },
     {
       "flag": "No error bars or significance tests",
       "detail": "All reported results are point estimates from apparently single runs. No statistical significance tests despite claims of improvement across multiple benchmarks and model sizes. With small test sets (300 samples), observed differences could be within noise."
     },
     {
       "flag": "LLM-as-judge circularity",
       "detail": "Gemini-2.0-flash is used as the LLM judge for both correctness verification during data generation AND final evaluation. The same model family (Gemini) generates the data, verifies it, and evaluates downstream results. No human validation of judge accuracy."
     },
     {
       "flag": "Unreleased code and data",
       "detail": "Code and data are promised for future release but not available. Results cannot be independently verified or reproduced."
     },
     {
       "flag": "Missing cost analysis",
       "detail": "The SAGE pipeline requires multiple rounds of Gemini API calls per training example (data generator + K search traces × R rounds), making it substantially more expensive than using existing datasets. This cost is never quantified or discussed relative to performance gains."
     }
   ],
   "cited_papers": [
     {
       "title": "Search-R1: Training LLMs to Reason and Leverage Search Engines with Reinforcement Learning",
       "authors": ["B. Jin", "H. Zeng", "Z. Yue"],
       "year": 2025,
       "relevance": "RL framework for training search agents, used as the training framework in downstream evaluation."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig"],
       "year": 2024,
       "relevance": "Major code generation benchmark referenced as an example of agentic LLM tasks."
     },
     {
       "title": "Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection",
       "authors": ["A. Asai", "Z. Wu", "Y. Wang"],
       "year": 2024,
       "relevance": "Retrieval-augmented generation with self-reflection, relevant to agentic RAG approaches."
     },
     {
       "title": "ReACT: Synergizing Reasoning and Acting in Language Models",
       "authors": ["S. Yao", "J. Zhao", "D. Yu"],
       "year": 2023,
       "relevance": "Foundational framework for interleaving reasoning and tool use that SAGE builds upon."
     },
     {
       "title": "GAIA: A Benchmark for General AI Assistants",
       "authors": ["G. Mialon", "C. Fourrier", "C. Swift"],
       "year": 2023,
       "arxiv_id": "2311.12983",
       "relevance": "General AI assistant benchmark used for Google Search transfer evaluation."
     },
     {
       "title": "Browsecomp: A Simple Yet Challenging Benchmark for Browsing Agents",
       "authors": ["J. Wei", "Z. Sun", "S. Papay"],
       "year": 2025,
       "relevance": "Deep search benchmark for browsing agents used for evaluation."
     },
     {
       "title": "Humanity's Last Exam",
       "authors": ["L. Phan"],
       "year": 2025,
       "arxiv_id": "2501.14249",
       "relevance": "Challenging evaluation benchmark used for search-subset evaluation."
     },
     {
       "title": "WebGPT: Browser-assisted Question-Answering with Human Feedback",
       "authors": ["R. Nakano", "J. Hilton", "S. Balaji"],
       "year": 2021,
       "relevance": "Foundational work on training search agents with human feedback."
     },
     {
       "title": "Fact, Fetch, and Reason: A Unified Evaluation of Retrieval-Augmented Generation",
       "authors": ["S. Krishna", "K. Krishna", "A. Mohananey"],
       "year": 2025,
       "doi": "10.18653/v1/2025.naacl-long.243",
       "relevance": "FRAMES benchmark used for out-of-domain evaluation of deep search agents."
     },
     {
       "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents",
       "authors": ["S. Zhou", "F. F. Xu", "H. Zhu"],
       "year": 2024,
       "relevance": "Web environment benchmark for autonomous agents."
     },
     {
       "title": "Qwen2.5 Technical Report",
       "authors": ["Q. A. Yang"],
       "year": 2024,
       "relevance": "Base model (Qwen-2.5-3B/7B-Instruct) used for training search agents in downstream evaluation."
     },
     {
       "title": "Proximal Policy Optimization Algorithms",
       "authors": ["J. Schulman", "F. Wolski", "P. Dhariwal"],
       "year": 2017,
       "relevance": "PPO algorithm used for reinforcement learning training of search agents."
     }
   ]
 }