ai-research-survey

scan.json (25489B)

---

 {
   "paper": {
     "title": "DatasetResearch: Benchmarking Agent Systems for Demand-Driven Dataset Discovery",
     "authors": [
       "Keyu Li",
       "Mohan Jiang",
       "Dayuan Fu",
       "Yunze Wu",
       "Xiangkun Hu",
       "Dequan Wang",
       "Pengfei Liu"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2508.06960",
     "doi": "10.48550/arXiv.2508.06960"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "DATASETRESEARCH benchmark with 208 real-world dataset demands reveals a stark performance gap: even advanced deep research systems achieve only 22% on the challenging pro subset. Search agents excel at knowledge-based tasks (42% fine-tuning score) while synthesis agents dominate reasoning tasks (73%). All current methods catastrophically fail on corner cases outside existing data distributions.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/GAIR-NLP/DatasetResearch, referenced in the abstract."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data is stated as 'publicly available' at the GitHub link. The 208 demands and metadata are released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment setup section is provided in the paper. Only a LlamaFactory config (Appendix D) is shown."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The LlamaFactory config in Appendix D is partial but doesn't constitute full reproduction instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2 and Figure 5 report point estimates only. No confidence intervals, error bars, or ± notation anywhere in the results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims search agents 'excel' at knowledge tasks and synthesis agents are 'superior' at reasoning tasks, but no statistical tests support these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported as normalized scores with baselines (e.g., GPT-4o-search 42% vs o3 w/ref 73% for reasoning fine-tuning), providing enough context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 208 tasks, 20 for the pro subset, or 500 synthesis samples. The pro subset selection is based on GPT-4o-search difficulty but the size is arbitrary."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures reported for any results. Single-run numbers throughout."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines: GPT-4o-search, GPT-4o-mini-search, OpenAI o3 (w/ and w/o ref), and deep research agents (OpenAI, Grok, Gemini). Zero-shot LLaMA-3.1-8B as floor."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4o-search-preview, OpenAI o3, OpenAI Deep Research, Gemini Deep Research, and Grok Deep Research — all state-of-the-art at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study of the benchmark design or evaluation framework components. The w/ref vs w/o ref comparison is a partial ablation of one variable but not a systematic ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Six metrics used across task types: Accuracy, F1-Score, Exact Match, BLEU, SacreBLEU, ROUGE (Table 1). Also metadata evaluation scores."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human verification is used in benchmark curation (Step 6) but no human evaluation of the agent systems' outputs. Metadata evaluation uses o3 as judge."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Reference datasets serve as held-out test sets. Fine-tuned models are evaluated on reference sets they were not trained on. Gated datasets prevent search agents from accessing originals."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by knowledge vs reasoning tasks (Table 2), per-metric metadata scores across 6 dimensions, and per-method comparisons in Figure 5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 discusses corner cases where all methods fail (Figure 8, medical coding example). Section 6.1-6.2 provide qualitative case studies."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that all methods perform poorly overall (max 22% on pro subset), and explicitly discusses failure modes across all agent types."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 22% max on pro subset, search/synthesis dichotomy, and corner case failures are all supported by Tables 2, Figure 5, and Section 6 analysis."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims synthesis agents' 'advantage stems from their ability to generate reasoning-rich, more instruction-aligned output data' — this is a causal claim from observational comparison without controlled experiment isolating this factor."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims this benchmarks 'AI agents' ability to discover and synthesize datasets' generally, but results are limited to NLP text tasks with LLaMA-3.1-8B as the only evaluation model. Title claims 'demand-driven dataset discovery' broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the search/synthesis performance gap. Could be prompt design, dataset formatting by o3, or LLaMA-3.1-8B's specific characteristics rather than fundamental agent properties."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures fine-tuning performance on LLaMA-3.1-8B as a proxy for 'dataset quality' and 'dataset discovery capability' without discussing whether this proxy captures what matters. A dataset could be valuable for other models or purposes."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models referred to by marketing names: 'GPT-4o-search-preview', 'OpenAI o3', 'Grok Deep Research'. No API versions, snapshot dates, or exact model identifiers provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts provided in Appendix C: demand description generation (C.1), search agent (C.2), synthesis agent (C.2), fine-tuning data extraction (C.2), metadata generation and evaluation (C.3)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides full LlamaFactory fine-tuning config including learning rate (1e-5), batch size, epochs (3), cutoff length (4096), scheduler, etc. Synthesis uses 10 samples × 50 iterations."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates third-party deep research agents (OpenAI, Grok, Gemini) as black boxes. The search and synthesis agents are simple prompt-based workflows, not agentic scaffolds."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents 7 filtering steps with counts: 1M+ → 422 (task/modality filter) → 261 (documentation check) → 104 (fine-tuning suitability) → 91 HuggingFace + 117 PapersWithCode = 208."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 'Limitation and Future Work' discusses three specific limitations: web-scale curation, open-source model evaluation, and hybrid agent approaches."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 7.2 discusses future work directions rather than specific threats to the current study's validity. No mention of LLM-as-judge bias, single evaluation model limitation, or prompt sensitivity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 7.2 mentions the benchmark 'relies on datasets from structured repositories' but does not explicitly bound claims. The paper's sweeping language ('finding any dataset in the digital universe') is not bounded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark and data are stated as publicly available at the GitHub repo. Reference datasets, demands, and metadata are released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 details 7-step collection pipeline from HuggingFace API filtering through human verification to metadata generation."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sourced from HuggingFace and PapersWithCode public repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented in Section 3.2 and Figure 2, with counts at each stage and filtering criteria described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations listed: Shanghai Jiao Tong University, SII, GAIR."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training cutoff dates stated for any of the evaluated models (GPT-4o, o3, LLaMA-3.1-8B, etc.)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 Step 1 explicitly uses 'gated' datasets to mitigate data leakage, as 'search agents cannot automatically download and process these datasets even if they are identified.' This is a partial but deliberate mitigation."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While gated datasets prevent download, the metadata/README files of these datasets may be in model training data. The paper doesn't discuss whether models have seen the dataset descriptions, names, or samples through their training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or per-task costs reported despite heavy use of o3 for metadata generation, evaluation, and synthesis (50 iterations × 208 tasks)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget stated. Full-parameter fine-tuning of LLaMA-3.1-8B is performed but GPU hours and hardware are not specified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds for fine-tuning experiments. Results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Number of experimental runs not stated anywhere. Cannot determine if results are single-run or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Appendix D shows 'fixed hyperparameters' for fine-tuning, but no discussion of whether these were tuned or how they were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'fixed hyperparameters' but doesn't justify why these specific values were chosen or whether any tuning was performed."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests performed at all, so no multiple comparison correction either."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the benchmark and evaluation framework, then evaluated agents on it. No discussion of potential bias in benchmark design favoring certain agent types."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Deep research agents likely use far more compute than simple search queries, but no compute-normalized comparison is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether fine-tuning LLaMA-3.1-8B performance actually measures 'dataset discovery quality.' The proxy relationship between fine-tuning success and dataset utility is not examined."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates different agent paradigms (search vs synthesis vs deep research) as distinct approaches rather than comparing models within a shared scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluated models were trained on data that includes the reference datasets' documentation, samples, or metadata."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The demand descriptions are generated from reference dataset metadata by o3. No discussion of whether these descriptions leak specific information that would make discovery trivial."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between the datasets used for fine-tuning and the reference test sets, or between o3's training data and the benchmark datasets."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Gated dataset selection (Step 1) is a concrete leakage prevention method — restricting to datasets that require manual approval prevents automatic download by search agents."
       }
     }
   },
   "claims": [
     {
       "claim": "Even advanced deep research systems achieve only 22% score on DatasetResearch-pro subset.",
       "evidence": "Figure 5 shows OpenAI DeepResearch achieving 0.2218 fine-tuning score on the pro subset (Section 5.2).",
       "supported": "strong"
     },
     {
       "claim": "Search agents excel at knowledge-based tasks while synthesis agents dominate reasoning-based tasks.",
       "evidence": "Table 2: GPT-4o-search achieves 42% fine-tuning on knowledge tasks; OpenAI o3 w/ref achieves 73% on reasoning tasks.",
       "supported": "moderate"
     },
     {
       "claim": "Few-shot evaluation results are closely aligned with fine-tuning experiments, making few-shot a viable preliminary assessment.",
       "evidence": "Table 2 shows relative trends between few-shot and fine-tuning are consistent across methods (Section 5.2).",
       "supported": "weak"
     },
     {
       "claim": "All current methods fail on corner cases outside existing data distributions.",
       "evidence": "Figure 8 shows degraded performance on medical coding task. Section 6.3 discusses the limitation qualitatively.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "LLM-as-judge circularity",
       "detail": "OpenAI o3 is used to generate reference metadata, generate demand descriptions, synthesize datasets, format search results, AND judge metadata similarity scores. This creates circular dependencies where the judge favors its own outputs."
     },
     {
       "flag": "Single evaluation model",
       "detail": "All fine-tuning evaluation uses only LLaMA-3.1-8B. Results may not generalize to other model sizes or architectures. A dataset good for LLaMA-8B may not be good for other models."
     },
     {
       "flag": "No variance or repeated runs",
       "detail": "Fine-tuning results are reported without any variance measures, error bars, or repeated runs. Full-parameter fine-tuning of 8B models is known to be sensitive to random seeds."
     },
     {
       "flag": "Pro subset selection bias",
       "detail": "DatasetResearch-pro is defined as the 20 hardest tasks for GPT-4o-search-preview. This systematically biases the pro subset toward tasks where search fails, potentially inflating the relative advantage of synthesis methods."
     },
     {
       "flag": "Overclaiming in abstract",
       "detail": "Abstract claims about 'finding any dataset in the digital universe' and 'self-improving AI systems' significantly overreach the actual scope of evaluating NLP text datasets from HuggingFace/PapersWithCode."
     }
   ],
   "cited_papers": [
     {
       "title": "DataFinder: Scientific dataset recommendation from natural language descriptions",
       "authors": [
         "Vijay Viswanathan",
         "Luyu Gao",
         "Tongshuang Wu",
         "Pengfei Liu",
         "Graham Neubig"
       ],
       "year": 2023,
       "arxiv_id": "2305.16636",
       "relevance": "Prior work on automated dataset recommendation from natural language queries."
     },
     {
       "title": "Better synthetic data by retrieving and transforming existing datasets",
       "authors": [
         "Saumya Gandhi",
         "Ritu Gala",
         "Vijay Viswanathan",
         "Tongshuang Wu",
         "Graham Neubig"
       ],
       "year": 2024,
       "arxiv_id": "2404.14361",
       "relevance": "Dataset transformation techniques for repurposing existing datasets, a precursor to synthesis-based approaches."
     },
     {
       "title": "DeepResearcher: Scaling deep research via reinforcement learning in real-world environments",
       "authors": [
         "Yuxiang Zheng",
         "Dayuan Fu",
         "Xiangkun Hu"
       ],
       "year": 2025,
       "arxiv_id": "2504.03160",
       "relevance": "Deep research agent using RL, evaluated in this benchmark."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": [
         "Carlos E Jimenez",
         "John Yang",
         "Alexander Wettig"
       ],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLM coding agents on real-world tasks."
     },
     {
       "title": "SWE-smith: Scaling data for software engineering agents",
       "authors": [
         "John Yang",
         "Kilian Leret",
         "Carlos E Jimenez"
       ],
       "year": 2025,
       "arxiv_id": "2504.21798",
       "relevance": "Data scaling for SE agents, directly relevant to AI-assisted software engineering research."
     },
     {
       "title": "ScienceAgentBench: Toward rigorous assessment of language agents for data-driven scientific discovery",
       "authors": [
         "Ziru Chen",
         "Shijie Chen",
         "Yuting Ning"
       ],
       "year": 2024,
       "arxiv_id": "2410.05080",
       "relevance": "Benchmark for scientific discovery agents with rigorous evaluation methodology."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-bench and Chatbot Arena",
       "authors": [
         "Lianmin Zheng",
         "Wei-Lin Chiang",
         "Ying Sheng"
       ],
       "year": 2023,
       "relevance": "Foundational work on LLM-as-judge evaluation methodology used in this paper's metadata scoring."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": [
         "Long Ouyang",
         "Jeffrey Wu",
         "Xu Jiang"
       ],
       "year": 2022,
       "relevance": "RLHF instruction tuning methodology used as basis for the fine-tuning evaluation approach."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": [
         "Daya Guo",
         "Dejian Yang",
         "Haowei Zhang"
       ],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "RL-based reasoning model relevant to agentic AI capabilities."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": [
         "Rylan Schaeffer",
         "Brando Miranda",
         "Sanmi Koyejo"
       ],
       "year": 2023,
       "relevance": "Analysis of LLM scaling properties, cited for long-context attention limitations affecting few-shot results."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Benchmarks dataset discovery agents but doesn't provide a usable tool — practitioners can't directly apply this to their workflows."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The 22% ceiling for deep research systems is notable but 'AI struggles on hard benchmark' is a familiar narrative, not a contrarian finding."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk angle whatsoever."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict — straightforwardly evaluates systems without challenging specific company claims."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is public on GitHub but reproducing requires fine-tuning LLaMA-3.1-8B, multiple API keys, and significant compute."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From Shanghai Jiao Tong University/GAIR — recognized in NLP but not a household name in broader tech circles."
     }
   }
 }