ai-research-survey

scan.json (24952B)

---

 {
   "paper": {
     "title": "How Far Are We from Genuinely Useful Deep Research Agents?",
     "authors": [
       "Dingling Zhang",
       "He Zhu",
       "Jincheng Ren",
       "Kangqi Song",
       "Xinran Zhou",
       "Boyu Feng",
       "Shudong Liu",
       "Jiabin Luo",
       "Weihao Xie",
       "Zhaohui Wang",
       "Tianrui Qin",
       "King Zhu",
       "Yuqing Wang",
       "Qianben Chen",
       "Yuchen Eleanor Jiang",
       "Wei Wang",
       "Wangchunshu Zhou",
       "Jiaheng Liu"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.01948",
     "doi": "10.48550/arXiv.2512.01948"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided: https://github.com/OPPO-PersonalAI/FINDER_DEFT, listed in the abstract area as 'Code & Data'."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The same GitHub repository is listed for both code and data. The benchmark (FINDER) with 100 tasks and 419 checklists is indicated as released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency details are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no details on how to reproduce experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table I.3 reports standard deviations for MiroFlow stability analysis across three runs. However, the main results in Table 1 do not include error bars or CIs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used to support claims of performance differences between systems. Rankings are based on raw score comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance differences are reported with baseline context, e.g., Gemini at 50.95 vs. Perplexity at 41.62 on RACE overall, and percentage breakdowns for failure categories (e.g., 'over 39% of failures arise in content generation')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 100 tasks were chosen, or why ~1000 reports were analyzed. No power analysis or discussion of sample adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Table 1) are single-run numbers for most systems. Only MiroFlow has multi-run variance reported (Appendix I, 3 runs). Other models show no variance information."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 13 systems across proprietary APIs, open-source models, and agent frameworks. It also compares FINDER against the original DeepResearch Bench (DRB)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Gemini 2.5 Pro, O3, O4-Mini, Kimi K2, and other 2025-era systems. These are contemporary and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The DRB vs FINDER comparison (Section 4.3, Figure 4) serves as an ablation showing the effect of refined prompts and checklists on evaluation outcomes."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple evaluation frameworks used: RACE (4 sub-dimensions), FACT (2 metrics), Positive Taxonomy Metrics (3 dimensions), and Checklist Pass Rate."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Four domain experts evaluated report outputs from WebThinker and OpenManus. ICR was computed between human and LLM annotations (Table 3). Seven experts curated the benchmark tasks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For DEFT validation (Section 3.2.3), 36 execution records from WebThinker and OpenManus (not involved in taxonomy construction) were used as a held-out set to test theoretical saturation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-dimension breakdowns (RACE sub-scores, FACT sub-scores, per-category taxonomy scores). Figure 3 shows per-category failure distributions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides extensive case studies for each of the 14 failure categories, with specific task IDs, model sources, and detailed analysis of what went wrong."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents that current DRAs struggle with evidence integration and verification. Specific system weaknesses are reported, e.g., Kimi K2 and O4-Mini suffering 'sharp decline in generation scores' despite strong reasoning."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about 39% generation failures and 32% retrieval failures are supported by Figure 3 and the taxonomy analysis. Claims about evidence integration struggles are supported by Table 1 results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'reasoning resilience... is the key factor determining whether an agent can consistently produce high-quality deep research outcomes' (Insight 1) without adequate causal identification. The correlation matrix (Figure 6) shows associations but is presented as evidence for causal failure pathways."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title asks 'How Far Are We from Genuinely Useful Deep Research Agents?' — a broad claim. Results are based on 100 tasks from DeepResearch Bench, which covers specific domains. The paper does not bound generalizations to these specific task types."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for observed failure patterns. For example, differences could stem from prompt sensitivity, API version changes, or evaluation framework biases rather than fundamental agent limitations."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures checklist pass rates, RACE scores, and taxonomy metrics as proxies for 'genuine usefulness' of deep research agents but does not discuss the gap between these automated/semi-automated measures and actual real-world utility."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Some models use marketing names without exact versions: 'Gemini-2.5-Pro Deep Research', 'O3 Deep Research', 'O4-Mini Deep Research'. Open-source models specify some versions (e.g., 'WebThinker-QwQ-32B', 'AFM-45B') but proprietary APIs lack snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 provides example queries with full checklist items. Appendix D shows a failure analysis report example. Appendix G provides seed conceptual categories used for coding prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix K reports configurations: AFM (temperature=0.4, top_p=0.9, max_tokens=32K), MiroFlow (temperature=0.3, top_p=0.95, max_tokens=32K), OpenManus (temperature=0.0, max_tokens=8192). Proprietary APIs use 'default configurations'."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates agent frameworks (OWL, OpenManus, MiroFlow) but provides minimal description of their internal scaffolding. It says MiroFlow is a 'Dual-agent framework' and OWL is a 'Multi-agent architecture' without detailing the scaffolding that would explain performance differences."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents the benchmark construction pipeline: prompt refinement by 7 domain experts, checklist construction with iterative Gemini 2.5 Flash refinement, and independent quality evaluation. Section 3.2 details the coding pipeline with explicit stages."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section is present in the paper. The conclusion is brief and does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. Issues like LLM judge reliability, benchmark representativeness, or evaluation framework biases are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what settings, task types, or agent architectures are excluded from its claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository is indicated to contain the benchmark data and code. The 100 tasks and 419 checklists would enable verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes benchmark construction from DeepResearch Bench with expert refinement. Section 3.2 describes the grounded-theory-based taxonomy construction with detailed stages."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Seven domain experts and four annotators are mentioned but their recruitment, expertise domains, and potential selection biases are not described. We learn only that they are 'domain experts'."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table F.1 documents the full coding pipeline with generation counts, refinement rounds, and group partitioning. Algorithms 1-4 in Appendix F formalize each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The paper is by 'OPPO AI Agent Team' but no funding acknowledgments section exists."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper is identified as from 'OPPO AI Agent Team'. Correspondence addresses include @oppo.com and @nju.edu.cn."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OPPO is a technology company with potential commercial interest in deep research agent capabilities. They evaluate their own MiroFlow and MiroThinker systems alongside competitors, creating a non-independent evaluation. No disclosure of this conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. OPPO employees evaluating OPPO's MiroFlow/MiroThinker systems against competitors is an undisclosed financial interest."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper evaluates agents on report generation tasks, not pre-trained model knowledge on benchmarks. The benchmark tests synthesis/retrieval capabilities, not memorized answers."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Report generation tasks with web retrieval are not susceptible to traditional train/test contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The benchmark tests open-ended report generation with live web retrieval, not closed-form answers that could be memorized."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the traditional sense. Expert annotators are used for validation, not as study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Expert annotators are collaborators, not participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency are reported for any of the evaluated systems despite testing ~1000 report generations across 13 systems."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated for running the ~1000 report generations or the annotation/coding process."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only MiroFlow was run 3 times (Appendix I). All other 12 systems appear to be single-run evaluations with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Number of runs is stated only for MiroFlow (3 runs, Appendix I). For other systems, it is unclear whether results are single-run or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. It is unclear whether the reported configurations were tuned or are defaults."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why specific configurations were chosen. Appendix K states proprietary APIs used 'default configurations' but does not justify open-source model settings."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, let alone corrections for multiple comparisons across 13 systems and multiple metrics."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "OPPO evaluates its own MiroFlow and MiroThinker systems alongside competitors. No acknowledgment of self-comparison bias. MiroFlow achieves the highest checklist pass rate (72.19%), which is not discussed as a potential bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of compute budget differences between systems. Proprietary deep research APIs likely use vastly more compute than open-source models, but this is not analyzed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether FINDER's checklist-based evaluation actually measures 'genuine usefulness' of deep research agents as claimed in the title. The construct validity gap between checklist pass rates and real-world utility is not addressed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Systems are compared with different scaffolding (proprietary APIs with unknown scaffolding vs. open-source models vs. agent frameworks), but the scaffold confound is not addressed. Performance differences are attributed to the systems rather than isolating model vs. scaffold effects."
       }
     }
   },
   "claims": [
     {
       "claim": "Over 39% of DRA failures arise in content generation, particularly through strategic content fabrication.",
       "evidence": "Figure 3 shows generation category at 38.76%. SCF (Strategic Content Fabrication) is the largest single failure mode at 19.0%.",
       "supported": "strong"
     },
     {
       "claim": "Retrieval-related failures account for over 32% of errors.",
       "evidence": "Figure 3 shows retrieval category at 33.10%, with IIA (Insufficient External Information Acquisition) at 16.3% being the largest.",
       "supported": "strong"
     },
     {
       "claim": "Current DRAs struggle not with task comprehension but with evidence integration, verification, and reasoning-resilient planning.",
       "evidence": "Reasoning failures account for only 28.14% vs. retrieval 33.10% and generation 38.76%. FUR (requirements understanding) is only 10.55%.",
       "supported": "moderate"
     },
     {
       "claim": "DEFT achieves strong inter-coder reliability between human and LLM annotations.",
       "evidence": "Table 3 reports Krippendorff's alpha coefficients: OpenManus avg 0.8203, WebThinker avg 0.8526, both above the 0.80 threshold for strong reliability.",
       "supported": "strong"
     },
     {
       "claim": "FINDER enforces stronger constraints on reasoning transparency and source reliability than DRB.",
       "evidence": "Figure 4 shows FACT scores generally decline under FINDER compared to DRB, except O3 citation effectiveness. RACE scores remain similar.",
       "supported": "moderate"
     },
     {
       "claim": "The taxonomy achieved theoretical saturation.",
       "evidence": "Section 3.2.3: 36 held-out records from WebThinker and OpenManus were coded with no new categories emerging.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "The paper introduces FINDER, a 100-task benchmark with 419 structured checklists for evaluating deep research agents, and DEFT, a 14-category failure taxonomy built via grounded theory. Evaluation of 13 DRA systems shows generation failures dominate (39%), with strategic content fabrication being the most common single failure mode (19%). Gemini 2.5 Pro leads on RACE quality metrics while MiroFlow achieves the highest checklist pass rate. The taxonomy reveals that DRAs struggle less with task comprehension and more with evidence integration and verification.",
   "red_flags": [
     {
       "flag": "Conflict of interest: company evaluating its own product",
       "detail": "OPPO AI Agent Team evaluates its own MiroFlow and MiroThinker systems alongside competitors. MiroFlow-English achieves the highest checklist pass rate (72.19%). No conflict of interest is disclosed."
     },
     {
       "flag": "Single-run evaluations for most systems",
       "detail": "Only MiroFlow was evaluated across multiple runs (3). All other 12 systems appear to be single-run, making results subject to random variation in generation."
     },
     {
       "flag": "No significance testing",
       "detail": "Performance rankings are based on raw score comparisons without any statistical tests. Small differences between systems (e.g., Gemini 50.95 vs Kimi 48.28) may not be meaningful."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations, threats to validity, or scope boundaries despite making broad claims about the state of deep research agents."
     },
     {
       "flag": "LLM-as-judge circularity",
       "detail": "RACE uses Gemini 2.5 Pro as the judge LLM, and Gemini 2.5 Pro Deep Research is also one of the evaluated systems. FACT uses Gemini 2.5 Flash. The potential circularity of using Gemini to judge Gemini is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "GAIA: a benchmark for general AI assistants",
       "authors": ["Grégoire Mialon", "Clémentine Fourrier", "Thomas Wolf", "Yann LeCun", "Thomas Scialom"],
       "year": 2023,
       "relevance": "Major benchmark for evaluating general AI agent capabilities, relevant to agent evaluation methodology."
     },
     {
       "title": "Humanity's Last Exam",
       "authors": ["Long Phan"],
       "year": 2025,
       "arxiv_id": "2501.14249",
       "relevance": "Expert-level evaluation benchmark for AI, relevant to understanding benchmark difficulty and evaluation design."
     },
     {
       "title": "WebThinker: Empowering Large Reasoning Models with Deep Research Capability",
       "authors": ["Xiaoxi Li"],
       "year": 2025,
       "arxiv_id": "2504.21776",
       "relevance": "Open-source deep research agent evaluated in this study, relevant to agentic AI systems."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Failure taxonomy for multi-agent LLM systems, directly comparable to DEFT taxonomy."
     },
     {
       "title": "OWL: Optimized Workforce Learning for General Multi-Agent Assistance in Real-World Task Automation",
       "authors": ["Mengkang Hu"],
       "year": 2025,
       "arxiv_id": "2505.23885",
       "relevance": "Multi-agent framework evaluated in this study, relevant to agentic AI architecture."
     },
     {
       "title": "OpenManus: An Open-Source Framework for Building General AI Agents",
       "authors": ["Xinbin Liang"],
       "year": 2025,
       "relevance": "Open-source agent framework evaluated in this study."
     },
     {
       "title": "Mind2Web 2: Evaluating Agentic Search with Agent-as-a-Judge",
       "authors": ["Boyu Gou"],
       "year": 2025,
       "arxiv_id": "2506.21506",
       "relevance": "Benchmark for agentic search with agent-as-judge evaluation framework."
     },
     {
       "title": "DeepResearchGym: A Free, Transparent, and Reproducible Evaluation Sandbox for Deep Research",
       "authors": ["João Coelho"],
       "year": 2025,
       "arxiv_id": "2505.19253",
       "relevance": "Sandbox environment for evaluating deep research agents with reproducible protocols."
     },
     {
       "title": "Kimi K2: Open Agentic Intelligence",
       "authors": ["Kimi Team"],
       "year": 2025,
       "arxiv_id": "2507.20534",
       "relevance": "Major open agentic model evaluated in this study."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["Wangchunshu Zhou"],
       "year": 2023,
       "relevance": "Multi-agent framework for software development, relevant to agentic AI coordination."
     },
     {
       "title": "BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents",
       "authors": ["Jason Wei"],
       "year": 2025,
       "arxiv_id": "2504.12516",
       "relevance": "Benchmark for browsing agents, relevant to web-based agent evaluation."
     }
   ]
 }