ai-research-survey

scan-v5.json (23197B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "How Far Are We from Genuinely Useful Deep Research Agents?",
     "authors": [
       "Dingling Zhang",
       "He Zhu",
       "Jincheng Ren",
       "Kangqi Song",
       "Xinran Zhou"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2512.01948",
     "doi": "10.48550/arXiv.2512.01948"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All major abstract claims are backed by paper content: 100 tasks with 419 checklist items (Section 3.1), DEFT's 14 failure modes (Table 2), and the 39%/32% failure distribution figures are confirmed by Figure 3 and Table 2 experimental data.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-sounding claims (structured checklists 'enhance evaluation precision,' DRAs 'struggle with' specific capabilities) but the study design is observational benchmark evaluation without ablations or controlled experiments adequate for causal inference.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion broadly states 'current agents' struggle with evidence integration based on 13 evaluated systems and 100 tasks from a single source benchmark (DRB), without bounding these generalizations to the specific task types, domains, or conditions tested.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for observed failure patterns—for example, whether RACE score consistency between DRB and FINDER reflects task difficulty confounds, evaluator bias from using Gemini as both reference-generator and judge, or benchmark-specific artifacts.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes RACE (relative quality against AI reference), FACT (citation accuracy), and checklist pass rate as measuring different aspects; Section 4.3 explicitly notes RACE measures relative not absolute performance, showing awareness of proxy measurement limitations.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Methodological constraints are mentioned inline (RACE relativity in Section 4.3, FACT failures in Appendix M) but not consolidated into a formal limitations discussion.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are identified. The paper does not address whether expert checklist creation introduces systematic bias, whether Gemini-generated reference reports inflate Gemini's RACE scores, or whether the 100-task sample is representative of real deep research needs.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. There is no discussion of task types excluded from FINDER, DRA capabilities the benchmark cannot assess, or limits on how checklist scores translate to real-world research quality.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment appears anywhere in the paper. The work is attributed to OPPO AI Agent Team but no institutional funding, grants, or corporate research budget disclosure is provided.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicitly attributed to 'OPPO AI Agent Team' with correspondence to oppo.com and nju.edu.cn addresses, making the primary organizational affiliations identifiable.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OPPO/MiroMind-affiliated systems (MiroFlow, MiroThinker) are among the evaluated DRAs while OPPO employees constructed the benchmark, creating a potential conflict where benchmark designers may have inadvertently favored evaluation criteria aligned with their own systems' strengths.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is provided anywhere in the paper, despite the commercial context (OPPO is a major consumer electronics company) and the commercial DRA market being evaluated.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined: 'Deep Research Agents' in the introduction, RACE and FACT frameworks in Appendix E, 'grounded theory' with methodology citations, and all 14 DEFT failure categories are formally defined in Appendix B with detailed case studies.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions are enumerated at the end of the introduction: FINDER benchmark (100 tasks, 419 checklist items), DEFT failure taxonomy (14 modes across 3 dimensions), and experimental findings demonstrating current DRA weaknesses.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 explicitly compares FINDER against GAIA, HLE, Mind2Web 2, DeepResearchGym, DeepScholar-Bench, and DRBench, explaining how FINDER differs from each via structured checklists versus subjective metrics and expert-curated tasks versus LLM-generated tasks.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": false,
           "justification": "The paper asserts that expert-curated structured checklists improve evaluation of deep research quality but does not empirically validate that checklist scores correlate with independent human expert judgments of report quality; construct validity rests on expert authority alone.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "Figure J.3 shows only the distribution of checklist items per query (3, 4, or 5), not the difficulty distribution of tasks. No easy/medium/hard tiers are defined, and model score variance is not analyzed as a proxy for task difficulty.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "No ceiling or floor effect analysis is performed. Checklist pass rates range from 44.87% to 72.19%, RACE scores from ~30 to ~51—the range suggests no obvious ceiling, but this is not explicitly verified or discussed.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline is provided. Reference reports used in RACE evaluation are generated by Gemini-2.5-Pro Deep Research, not human researchers, so performance is benchmarked against an AI standard rather than human analyst capability.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": false,
           "justification": "RACE/FACT metrics are inherited from DeepResearch Bench without independent justification. The novel cosine-based positive taxonomy metric (Appendix L) is described mathematically but its choice over simpler linear alternatives is not adequately motivated beyond aesthetic preference for curvature.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "No contamination resistance measures are mentioned—no temporal splits, canary strings, or dynamic task generation. The 100 tasks derive from DeepResearch Bench PhD-level topics that may appear in model training data.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether FINDER will remain discriminative as DRA capabilities improve, whether benchmark tasks will be gamed, or any update plan for the task set or scoring criteria as the field advances.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "DEFT documents failure modes of DRAs, not of the benchmark itself. Appendix M addresses FACT technical failures (anti-scraping, fabricated URLs) but the paper does not systematically analyze what capabilities FINDER cannot measure or how agents could game it.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Code and data are released at github.com/OPPO-PersonalAI/FINDER_DEFT, and Appendix K provides detailed model configurations for all 13 evaluated systems, enabling others to reproduce reported numbers.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": false,
           "justification": "The paper describes construction methodology and provides query examples (Appendix A.2), but lacks a formal data card, does not detail preprocessing steps or quality control for LLM-based checklist refinement, and omits information about expert selection criteria.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "The paper links to a GitHub repository but does not specify the license under which FINDER tasks and DEFT annotations are released, leaving downstream use terms and redistribution rights unclear.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": false,
           "justification": "While the intended use (evaluating DRAs) is implicit, the paper does not specify what conclusions should NOT be drawn from benchmark results—such as whether FINDER scores generalize to real-world research utility or how domain imbalances affect score interpretation.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Over 39% of failures in deep research agents arise in content generation, particularly through strategic content fabrication (19%) where agents generate plausible but unsupported professional-looking content",
       "evidence": "Figure 3 and Table 2 report Generation failures at 38.76% with SCF as the single largest failure mode at 19.0%, derived from analysis of ~1,000 reports across 9 evaluated systems",
       "supported": "strong"
     },
     {
       "claim": "Current DRAs struggle not with task comprehension but with evidence integration, verification, and reasoning-resilient planning",
       "evidence": "Reasoning failures are lowest at 28.14% while Retrieval (33.10%) and Generation (38.76%) are higher; the paper explicitly frames this as 'reasoning resilience' vs. 'reasoning intensity' (Insight 1)",
       "supported": "moderate"
     },
     {
       "claim": "FINDER imposes stricter evaluation standards than DeepResearch Bench (DRB), exposing model weaknesses in factual consistency that DRB's original configuration obscures",
       "evidence": "Section 4.3 shows FACT scores diverge significantly between DRB and FINDER, with most systems experiencing declines in citation accuracy and effective citations under FINDER's harder prompts",
       "supported": "moderate"
     },
     {
       "claim": "Gemini-2.5-Pro Deep Research is the top-performing system with RACE overall score 50.95 and balanced performance across all taxonomy dimensions (Savg 72.89)",
       "evidence": "Table 1 shows Gemini leading on RACE Overall, and achieving the highest Savg among all evaluated proprietary API systems",
       "supported": "strong"
     },
     {
       "claim": "DEFT achieves strong inter-coder reliability with Krippendorff's alpha >0.8 between human expert and LLM coder pairs",
       "evidence": "Table 3 reports overall alpha of 0.8203 (OpenManus) and 0.8526 (WebThinker), with category-level coefficients ranging from 0.74 to 0.92",
       "supported": "strong"
     },
     {
       "claim": "Retrieval failures account for 32%+ of errors with insufficient external information acquisition (IIA) as the largest retrieval failure mode at 16.3%",
       "evidence": "Figure 3 and Table 2 report IIA at 16.30% as the second-largest single failure mode across all three dimensions",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "qualitative"
   ],
   "key_findings": "FINDER introduces 100 expert-curated deep research tasks with 419 structured checklist items that impose stricter evaluation than existing QA-based benchmarks, particularly for citation fidelity and analytical depth. DEFT identifies 14 fine-grained failure modes across reasoning, retrieval, and generation dimensions, finding that strategic content fabrication (19.0%) and insufficient information acquisition (16.3%) are the most prevalent failure types—contrary to the assumption that task comprehension is the main bottleneck, DRAs demonstrate adequate understanding but fail catastrophically at evidence grounding and fabrication avoidance. Proprietary systems substantially outperform open-source alternatives on most dimensions, with Gemini-2.5-Pro leading overall and O3 Deep Research excelling on factual citation metrics. The benchmark reveals that superior DRA performance requires maintaining balance across understanding, evidence collection, and synthesis rather than overoptimizing any single stage.",
   "red_flags": [
     {
       "flag": "Benchmark-evaluator conflict of interest",
       "detail": "OPPO/MiroMind-affiliated systems (MiroFlow, MiroThinker) are evaluated using a benchmark created by the same team, and no conflict-of-interest statement is provided. Benchmark criteria may inadvertently favor evaluation dimensions aligned with the creators' systems."
     },
     {
       "flag": "LLM-as-gold-standard",
       "detail": "RACE evaluation uses Gemini-2.5-Pro-generated reports as the reference standard, inherently biasing relative scores toward Gemini-similar outputs. The benchmark measures quality against an AI gold standard rather than human expert work."
     },
     {
       "flag": "No human baseline",
       "detail": "The benchmark claims to evaluate 'analyst-level' report generation but provides no human expert performance baseline, making it impossible to calibrate how far any system is from actual human capability or to validate the benchmark's discriminative ceiling."
     },
     {
       "flag": "Construct validity unvalidated",
       "detail": "Checklist item validity rests solely on expert authority. No external validation study shows that checklist scores correlate with independent human judgments of report quality or downstream research utility."
     },
     {
       "flag": "No competing interests disclosure",
       "detail": "No funding source or competing interests statement is provided despite OPPO's commercial context, the presence of affiliated systems among those evaluated, and potential commercial interests in the DRA market outcomes."
     },
     {
       "flag": "Arbitrary scoring metric",
       "detail": "The novel cosine-based positive taxonomy metric is post-hoc justified by analogy to cosine similarity; the choice over simpler alternatives (e.g., linear inverse error rate) is not driven by measurement validity arguments."
     }
   ],
   "cited_papers": [
     {
       "title": "GAIA: a benchmark for general AI assistants",
       "relevance": "Foundational DRA evaluation benchmark that FINDER positions against; represents the closed-ended QA paradigm the authors critique"
     },
     {
       "title": "DeepResearch Bench: A comprehensive benchmark for deep research agents",
       "relevance": "Direct predecessor that FINDER extends with structured checklists and refined prompts; FINDER's 100 tasks derive from DRB's task set"
     },
     {
       "title": "Mind2Web 2: Evaluating agentic search with agent-as-a-judge",
       "relevance": "Contemporary open-ended DRA benchmark using Agent-as-Judge framework; key comparison point for FINDER's evaluation approach"
     },
     {
       "title": "DeepResearchGym: A free, transparent, and reproducible evaluation sandbox for deep research",
       "relevance": "Contemporary DRA benchmark emphasizing reproducibility with standardized search APIs; positioned as complementary to FINDER"
     },
     {
       "title": "DRBench: A realistic benchmark for enterprise deep research",
       "relevance": "Enterprise-focused DRA benchmark using citation-grounded assessment; shares FINDER's focus on long-form analytical reports"
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "relevance": "Prior failure taxonomy work (Cemri et al.) whose seed concepts and findings directly inform DEFT's open coding process"
     },
     {
       "title": "WebThinker: Empowering large reasoning models with deep research capability",
       "relevance": "Open-source DRA system evaluated in FINDER experiments; also used as one of two validation systems for DEFT's theoretical saturation check"
     },
     {
       "title": "Deep research agents: A systematic examination and roadmap",
       "relevance": "Survey of DRA systems and capabilities that provides context for the benchmark's scope and the field's current state"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "FINDER provides DRA developers a structured evaluation tool with released code and data, directly applicable to benchmarking new systems against a multi-dimensional quality rubric."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the assumption that DRAs fail primarily at task understanding; demonstrates fabrication (not retrieval or comprehension) is the #1 failure mode, reframing the field's optimization target."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Strategic Content Fabrication at 19% of all failures—AI confidently generating plausible but unfounded academic content—raises concrete concerns about reliability of AI-generated research in high-stakes contexts."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Competitive comparison across Gemini, OpenAI, Perplexity, and open-source systems reveals clear performance gaps, but the framing is measured and academic rather than confrontational."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and data released at github.com/OPPO-PersonalAI/FINDER_DEFT enables practitioners to run FINDER evaluation on their own DRA systems immediately."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Evaluates Gemini-2.5-Pro, OpenAI O3/O4-Mini Deep Research, and Perplexity; uses Claude Opus 4.1 as one of the five LLM coders in taxonomy construction; OPPO is a globally recognized consumer electronics brand."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "38869223",
         "title": "Show HN: RAGatouille, a simple lib to use&train top retrieval models in RAG apps",
         "points": 15,
         "comments": 5,
         "url": "https://news.ycombinator.com/item?id=38869223",
         "created_at": "2024-01-04T16:48:34Z"
       },
       {
         "hn_id": "46439655",
         "title": "Stable-Pretraining-v1: Foundation Model Research Made Simple",
         "points": 7,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46439655",
         "created_at": "2025-12-30T23:51:39Z"
       },
       {
         "hn_id": "43063462",
         "title": "The hierarchy in HNSW is not necessary in high dimensions",
         "points": 5,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43063462",
         "created_at": "2025-02-15T23:19:07Z"
       },
       {
         "hn_id": "33870608",
         "title": "Clustering – Basic Concepts and Methods",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=33870608",
         "created_at": "2022-12-05T19:30:19Z"
       },
       {
         "hn_id": "45665498",
         "title": "A Fine-Grained Purpose-Based Access Control System for Large Data Warehouses",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45665498",
         "created_at": "2025-10-22T06:24:49Z"
       },
       {
         "hn_id": "42962673",
         "title": "Paradoxical Behavior in Collatz Sequences",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42962673",
         "created_at": "2025-02-06T14:33:11Z"
       },
       {
         "hn_id": "46504193",
         "title": "Evolution Without an Oracle: Driving Effective Evolution with LLM Judges",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46504193",
         "created_at": "2026-01-05T20:12:58Z"
       },
       {
         "hn_id": "42517812",
         "title": "Improving feature interactions at Pinterest under industry constraints",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42517812",
         "created_at": "2024-12-26T21:05:19Z"
       },
       {
         "hn_id": "29450974",
         "title": "ColBERTv2: Effective and Efficient Retrieval via Lightweight Late Interaction",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=29450974",
         "created_at": "2021-12-05T17:36:32Z"
       }
     ],
     "top_points": 15,
     "total_points": 38,
     "total_comments": 6
   }
 }