ai-research-survey

scan-v4.json (20469B)

---

 {
   "scan_version": 4,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "DatasetResearch: Benchmarking Agent Systems for Demand-Driven Dataset Discovery",
     "authors": [
       "Keyu Li",
       "Mohan Jiang",
       "Dayuan Fu",
       "Yunze Wu",
       "Xiangkun Hu"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2508.06960",
     "doi": "10.48550/arXiv.2508.06960"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 22% max on pro subset, search/synthesis dichotomy, and corner case failures are all supported by Tables 2, Figure 5, and Section 6 analysis.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims synthesis agents' 'advantage stems from their ability to generate reasoning-rich, more instruction-aligned output data' — this is a causal claim from observational comparison without controlled experiment isolating this factor.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims this benchmarks 'AI agents' ability to discover and synthesize datasets' generally, but results are limited to NLP text tasks with LLaMA-3.1-8B as the only evaluation model. Title claims 'demand-driven dataset discovery' broadly.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the search/synthesis performance gap. Could be prompt design, dataset formatting by o3, or LLaMA-3.1-8B's specific characteristics rather than fundamental agent properties.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures fine-tuning performance on LLaMA-3.1-8B as a proxy for 'dataset quality' and 'dataset discovery capability' without discussing whether this proxy captures what matters. A dataset could be valuable for other models or purposes.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 'Limitation and Future Work' discusses three specific limitations: web-scale curation, open-source model evaluation, and hybrid agent approaches.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 7.2 discusses future work directions rather than specific threats to the current study's validity. No mention of LLM-as-judge bias, single evaluation model limitation, or prompt sensitivity.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 7.2 mentions the benchmark 'relies on datasets from structured repositories' but does not explicitly bound claims. The paper's sweeping language ('finding any dataset in the digital universe') is not bounded.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations listed: Shanghai Jiao Tong University, SII, GAIR.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 formally defines 'data discovery,' 'DataResearcher,' and 'MetaTriplet'; knowledge-based vs. reasoning-based categories are defined with explicit criteria in Section 3.2.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Four explicit bullet-point contributions are listed at the end of Section 1: the benchmark itself, the evaluation methodology, experimental findings, and failure mode analysis.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 explicitly positions DATASETRESEARCH against Viswanathan et al. 2023, Walker et al. 2023, and Gandhi et al. 2024, explaining why prior approaches 'do not fully harness the deep reasoning and inferential capabilities enabled by modern LLMs.'",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": false,
           "justification": "The paper asserts the benchmark measures 'demand-driven dataset discovery capability' but never argues why downstream LLaMA-3.1-8B fine-tuning performance on 6 NLP tasks is a valid proxy for this capability — the connection between the proxy metric and the claimed construct is assumed rather than justified.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "The only difficulty characterization is a binary knowledge/reasoning split and a post-hoc hard subset (DatasetResearch-pro) selected by GPT-4o-search worst-performing tasks; no formal difficulty tiers, IRT analysis, or difficulty measurement is provided.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": true,
           "justification": "Step 2 of curation explicitly excludes tasks 'where baseline model performance had already reached near-saturation levels,' and the pro subset was created specifically because standard tasks showed insufficient discrimination.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline is reported; while deep research agents required human-in-the-loop execution, the humans' performance as data researchers is not measured or reported.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": false,
           "justification": "The normalized score formula (Seval/Sref) is explained pragmatically for cross-task comparability, but the choice to use LLaMA-3.1-8B as the evaluation model is unjustified, and the metadata scoring by o3 is circular since o3 also generated the reference metadata.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "The benchmark explicitly selects 'gated' HuggingFace datasets requiring manual approval for access, directly preventing search agents from automatically downloading reference datasets even if identified.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Future work mentions extending to unstructured web data but does not discuss whether or how the benchmark will be updated as model capabilities evolve or whether current tasks will become trivial.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 6.3 explicitly analyzes 'corner cases' as a failure mode of all methods, explaining that both search and synthesis are fundamentally limited by training data distributions.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "The GitHub repository (GAIR-NLP/DatasetResearch) is linked, search/synthesis/deep-research agent implementations are described in detail with prompts in Appendix C, and fine-tuning configs are provided in Appendix D.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "The 7-step curation pipeline for HuggingFace datasets is documented in detail, metadata schema (6 dimensions) is specified in Appendix A, and collection methodology from PapersWithCode is described analogously.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "The GitHub URL is provided but no license for the benchmark itself is stated; the reference datasets remain gated/restricted, which limits replicability without separately obtaining HF access to each underlying dataset.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.1 clearly defines the intended use (evaluating AI agents on demand-driven dataset discovery and synthesis), and limitations Section 7.2 bounds the scope to NLP text tasks from structured repositories.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Even the most advanced deep research systems achieve only 22% score on the DatasetResearch-pro subset.",
       "evidence": "Figure 5 and Section 5.2 report OpenAI DeepResearch achieving 0.2218 on DatasetResearch-pro across few-shot and fine-tuning settings.",
       "supported": "strong"
     },
     {
       "claim": "Search agents excel at knowledge-based tasks while synthesis agents dominate reasoning-based tasks.",
       "evidence": "Table 2 shows GPT-4o-search achieves 41.89% fine-tuning on knowledge vs. 27.54% on reasoning; OpenAI o3 w/ ref achieves 38.98% on knowledge vs. 72.70% on reasoning.",
       "supported": "strong"
     },
     {
       "claim": "All current agent methodologies fail on corner cases outside existing data distributions.",
       "evidence": "Section 6.3 presents a case study where all methods (search, synthesis, deep research) score near 0% on a niche medical coding dataset, compared to a 91.7% reference ceiling.",
       "supported": "moderate"
     },
     {
       "claim": "Few-shot evaluation (especially 3-shot) serves as a reliable proxy for fine-tuning performance.",
       "evidence": "Section 5.2 observes 'consistent relative trends' between few-shot and fine-tuning results; 3-shot is claimed most stable, but no correlation coefficient is reported.",
       "supported": "weak"
     },
     {
       "claim": "Providing a reference sample to the synthesis agent (o3 w/ ref) improves performance over w/o ref.",
       "evidence": "Table 2 shows fine-tuning scores of 72.70% (w/ ref) vs. 67.25% (w/o ref) on reasoning tasks and 38.98% vs. 37.94% on knowledge tasks.",
       "supported": "moderate"
     },
     {
       "claim": "Deep research methodology retrieves significantly higher quality data than single-shot search agents on DatasetResearch-pro.",
       "evidence": "Figure 5 shows OpenAI DeepResearch outperforming GPT-4o-mini-search on DatasetResearch-pro; Figure 7 case study shows DeepResearch (0.19) vs. search (0.005) and synthesis (0.05) on a Vietnamese exam task.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "DATASETRESEARCH establishes the first benchmark for demand-driven dataset discovery, revealing that current AI agents fall far short of the goal: even the best deep research systems score only 22% on the hard DatasetResearch-pro subset. A fundamental performance dichotomy emerges: search agents excel at knowledge-intensive tasks via retrieval breadth while synthesis agents dominate reasoning tasks via structured generation, yet both fail catastrophically on corner cases outside existing data distributions. The benchmark demonstrates that automated data curation remains an unsolved problem, with fine-tuning on discovered datasets achieving at most 73% of the reference performance on reasoning tasks and 42% on knowledge tasks.",
   "red_flags": [
     {
       "flag": "Circular metadata evaluation",
       "detail": "OpenAI o3 generates the reference metadata and also serves as the judge for metadata similarity scoring — the paper claims this 'systematically mitigates potential scoring biases' but the opposite is likely true: the judge will reward data matching o3's generation style."
     },
     {
       "flag": "Post-hoc hard subset selection",
       "detail": "DatasetResearch-pro is constructed by selecting the 20 tasks where GPT-4o-search performed worst in preliminary experiments, then advanced systems are evaluated on this same subset — this introduces selection bias toward tasks that specifically challenge retrieval-based approaches."
     },
     {
       "flag": "Human-in-the-loop evaluation inconsistency",
       "detail": "Deep research tools required manual human execution and curation of results, introducing uncontrolled human judgment into the evaluation process that is not present for search and synthesis agents."
     },
     {
       "flag": "Single small evaluation model",
       "detail": "All downstream task performance is measured by fine-tuning LLaMA-3.1-8B specifically, with no validation that results generalize to other model families or scales."
     },
     {
       "flag": "Overreaching benchmark scope claims",
       "detail": "The abstract claims to 'illuminate the path toward AI systems capable of finding any dataset in the digital universe' based on 208 tasks across only 6 NLP categories from two structured repositories — this is a severe generalization overreach."
     },
     {
       "flag": "No human performance baseline",
       "detail": "The benchmark lacks any measurement of how humans perform as DataResearchers, making it impossible to contextualize agent performance relative to human capability on the same tasks."
     }
   ],
   "cited_papers": [
     {
       "title": "DataFinder: Scientific Dataset Recommendation from Natural Language Descriptions",
       "relevance": "Direct predecessor work on dataset discovery via natural language queries; DATASETRESEARCH explicitly builds beyond this approach"
     },
     {
       "title": "Prompting Datasets: Data Discovery with Conversational Agents",
       "relevance": "Prior work on conversational AI for data discovery; identified LLM hallucination of non-existent datasets as a key challenge"
     },
     {
       "title": "Better Synthetic Data by Retrieving and Transforming Existing Datasets",
       "relevance": "Prior work on dataset synthesis/transformation that DATASETRESEARCH extends with more rigorous agent evaluation"
     },
     {
       "title": "ScienceAgentBench: Toward Rigorous Assessment of Language Agents for Data-Driven Scientific Discovery",
       "relevance": "Related benchmark for scientific agent evaluation; helps situate this work in the agent benchmarking landscape"
     },
     {
       "title": "DiscoveryBench: Towards Data-Driven Discovery with Large Language Models",
       "relevance": "Related benchmark for LLM-assisted scientific discovery from data"
     },
     {
       "title": "DeepResearcher: Scaling Deep Research via Reinforcement Learning in Real-World Environments",
       "relevance": "Deep research agent methodology directly evaluated in this benchmark"
     },
     {
       "title": "SWE-Smith: Scaling Data for Software Engineering Agents",
       "relevance": "Example of dataset construction for agent training; motivates demand-driven dataset discovery"
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "relevance": "Foundational work for LLM-as-judge evaluation methodology used in metadata scoring"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Benchmarks dataset discovery agents but doesn't provide a usable tool — practitioners can't directly apply this to their workflows."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The 22% ceiling for deep research systems is notable but 'AI struggles on hard benchmark' is a familiar narrative, not a contrarian finding."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk angle whatsoever."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict — straightforwardly evaluates systems without challenging specific company claims."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is public on GitHub but reproducing requires fine-tuning LLaMA-3.1-8B, multiple API keys, and significant compute."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From Shanghai Jiao Tong University/GAIR — recognized in NLP but not a household name in broader tech circles."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "43014573",
         "title": "Time to act on the risk of efficient personalized text generation",
         "points": 57,
         "comments": 34,
         "url": "https://news.ycombinator.com/item?id=43014573",
         "created_at": "2025-02-11T16:14:03Z"
       },
       {
         "hn_id": "45234790",
         "title": "Reverse-Engineered Reasoning for Open-Ended Generation",
         "points": 4,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=45234790",
         "created_at": "2025-09-13T19:49:08Z"
       },
       {
         "hn_id": "45184326",
         "title": "Reasoning Traces from QA Pairs",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=45184326",
         "created_at": "2025-09-09T16:25:03Z"
       },
       {
         "hn_id": "44516439",
         "title": "Amazon gets serious with AI Safety",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44516439",
         "created_at": "2025-07-10T01:50:50Z"
       },
       {
         "hn_id": "45226714",
         "title": "Are ArXiv submissions on Wednesday better cited?",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45226714",
         "created_at": "2025-09-12T21:00:07Z"
       },
       {
         "hn_id": "44889206",
         "title": "Large Language Models Do Not Simulate Human Psychology",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44889206",
         "created_at": "2025-08-13T14:50:01Z"
       },
       {
         "hn_id": "32619543",
         "title": "Angle-agnostic cloaking from person-tracking systems with a t-shirt",
         "points": 1,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=32619543",
         "created_at": "2022-08-27T14:42:49Z"
       },
       {
         "hn_id": "44521323",
         "title": "Evaluating the Critical Risks of Amazon’s Nova Premier",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44521323",
         "created_at": "2025-07-10T14:11:52Z"
       },
       {
         "hn_id": "42705257",
         "title": "What Hawking Radiation Looks Like as You Fall into a Black Hole",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42705257",
         "created_at": "2025-01-14T23:16:02Z"
       }
     ],
     "top_points": 57,
     "total_points": 73,
     "total_comments": 37
   }
 }