ai-research-survey

scan.json (19840B)

---

 {
   "paper": {
     "title": "From Fluent to Verifiable: Claim-Level Auditability for Deep Research Agents",
     "authors": ["Razeen A Rasheed", "Somnath Banerjee", "Animesh Mukherjee", "Rima Hazra"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.13855"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["theoretical"],
   "key_findings": "The paper argues that deep research agents suffer from three systematic failure modes — objective drift, transient constraints, and unverifiable inference chains — that make their outputs scientifically unauditable. It proposes the Auditable Autonomous Research (AAR) standard with four metrics: provenance coverage, provenance soundness, contradiction transparency, and audit effort. The paper advocates semantic provenance graphs with protocolized validation as the architectural substrate for auditable research agents. Failure statistics are drawn entirely from prior evaluations (e.g., 44.2% planning failures, 42% experiment execution failures in The AI Scientist).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or implementation is referenced. The paper proposes a framework but provides no implementation artifacts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset or supplementary data is released. The paper draws on secondary sources but provides no novel data."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical/position paper with no computational experiments requiring an environment."
       },
       "reproduction_instructions": {
         "applies": false,
         "answer": false,
         "justification": "No experiments to reproduce. The contribution is a conceptual framework and measurement definitions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "No original experiments. All statistics cited (e.g., 44.2% failure rate, 42% execution failure) are from prior work."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative claims based on original data. This is a position paper citing others' results."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No original measurements are made. All quantitative claims reference prior evaluations."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No original data collection or sampling. Theoretical paper."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No original experiments with runs to report variance over."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "No empirical evaluation is conducted. The paper proposes a framework without experimental validation."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No baselines — no experiments."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system to ablate. This is a conceptual framework paper."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No empirical evaluation. The AAR metrics (PCov, PSnd, CTran, AEff) are proposed but not measured on any system."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No evaluation of any kind is conducted."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No data splits — no experiments."
       },
       "per_category_breakdown": {
         "applies": false,
         "answer": false,
         "justification": "No empirical results to break down."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 provides a detailed taxonomy of failure cases from prior work: planning failures (objective drift, metric misalignment, novelty verification), execution failures (cross-validation bug, transient memory), and synthesis failures (citation decorrelation, unverifiable inference). Figure 2 summarizes these."
       },
       "negative_results_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments that could produce positive or negative results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are appropriately scoped as a 'perspective' paper. It proposes the AAR standard and identifies failure modes, both of which are developed in the body. No unsupported empirical claims in the abstract."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims (e.g., 'cosine similarity... making it mathematically incapable of representing entailment or contradiction' as a cause of citation decorrelation, Section 3.3) without controlled experiments. These are argued from first principles rather than demonstrated experimentally."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes broad claims about 'deep research agents' but its failure analysis draws primarily from evaluations of The AI Scientist and ChemCrow. The generalization from these specific systems to all deep research agents is not explicitly bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 ('Alternative views and objections') addresses four counterarguments: bigger models will solve this, graphs are too expensive, logs are sufficient, and validation adds prohibitive latency. Each is engaged substantively."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical paper with no measurements. The proposed metrics (PCov, PSnd, CTran, AEff) are defined but not measured."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models are used. This is a theoretical/position paper."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in this paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used or built in this paper."
       },
       "data_preprocessing_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data preprocessing — no data collected or processed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. Section 6 addresses objections but does not discuss limitations of the authors' own proposal (e.g., feasibility of building the proposed provenance graphs, scalability of the AAR metrics)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats-to-validity discussion. The paper does not acknowledge that its failure taxonomy is derived from a small number of evaluated systems, nor that the proposed metrics are untested."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its framework does NOT address. It presents the AAR standard as broadly applicable without stating scope limitations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": false,
         "answer": false,
         "justification": "No original data collected. All evidence comes from published prior work."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper draws statistics from prior work (e.g., '44.2% of failures' from [19], '42% of experiments failed' from [32]) but does not systematically describe how these sources were identified or selected. The literature coverage appears opportunistic rather than systematic."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no benchmark data. Standard sources from published literature."
       },
       "data_pipeline_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data pipeline — this is a position paper synthesizing existing literature."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Indian Institute of Science, IIT Kharagpur, Cisco Systems, TCG CREST."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed. One author is affiliated with Cisco Systems but there is no disclosure of whether this creates a conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical paper with no method that incurs inference cost."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "No computation performed."
       }
     }
   },
   "claims": [
     {
       "claim": "44.2% of multi-agent system failures arise from specification errors, task misinterpretation, and improper decomposition during planning.",
       "evidence": "Cited from [19] (Cemri et al. 2025), analysis of ~1,642 multi-agent system traces. Section 2.1 and Figure 1.",
       "supported": "moderate"
     },
     {
       "claim": "Execution failure rates in multi-agent systems range from 41% to 86.7% depending on system architecture and task complexity.",
       "evidence": "Cited from [19] across multi-agent traces. Section 2.1.",
       "supported": "moderate"
     },
     {
       "claim": "42% of The AI Scientist's proposed experiments failed to execute due to coding errors, yet the system produced manuscripts mischaracterizing results.",
       "evidence": "Cited from [8] (Beel & Kan 2025) and [32] (evaluation of AI Scientist). Section 1 and 3.2.",
       "supported": "moderate"
     },
     {
       "claim": "Cosine similarity is mathematically incapable of representing entailment or contradiction because it is symmetric and blending.",
       "evidence": "Argued from first principles in Section 3.3, citing [16] (Bowman et al. 2015) and [69] (Wang et al. 2024). This is a theoretical argument, not empirically demonstrated in this paper.",
       "supported": "moderate"
     },
     {
       "claim": "Current deep research agents provide no reconstructible trace linking generated claims to supporting evidence through explicit reasoning steps.",
       "evidence": "Argued based on architectural analysis in Section 3.3, with references to [62] (PROV-AGENT) and [12]. No original empirical verification.",
       "supported": "weak"
     },
     {
       "claim": "Verification effort must be substantially lower than generation effort for autonomous systems to provide practical value.",
       "evidence": "Stated as a design principle in Section 3.3, referencing [26] and [37]. Formalized as the auditability invariant in Definition 1 (Section 4.1).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No empirical validation of proposed framework",
       "detail": "The AAR standard (PCov, PSnd, CTran, AEff) is proposed with formal definitions and a worked example but never applied to any real system. The paper does not demonstrate that these metrics are measurable, discriminative, or actionable in practice."
     },
     {
       "flag": "Narrow evidence base generalized broadly",
       "detail": "The failure taxonomy draws heavily from evaluations of The AI Scientist (refs [8], [32]) and to a lesser extent ChemCrow [17]. These are generalized to all 'deep research agents' without acknowledging the narrow empirical base."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper engages objections (Section 6) but does not discuss limitations of its own proposal, such as the feasibility of implementing semantic provenance graphs at scale, the cost of NLI-based entailment checking, or whether the AAR metrics have construct validity."
     },
     {
       "flag": "Selective citation of failure statistics",
       "detail": "Failure rates are presented from prior work without discussing the conditions under which they were measured or whether they generalize. For example, '28.6-91.4% made-up information' in Figure 1 is attributed without context on which systems or tasks produced these rates."
     }
   ],
   "cited_papers": [
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": ["Mert Cemri", "Melissa Z. Pan", "Shuyi Yang", "Lakshya A. Agrawal"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Empirical analysis of 1,642 multi-agent system traces identifying systematic failure modes — directly relevant to agentic AI reliability."
     },
     {
       "title": "The AI Scientist: Towards fully automated open-ended scientific discovery",
       "authors": ["Chris Lu", "Cong Lu", "Robert Tjarko Lange", "Jakob Foerster", "Jeff Clune", "David Ha"],
       "year": 2024,
       "arxiv_id": "2408.06292",
       "relevance": "Primary autonomous research agent system evaluated throughout this paper; key subject of failure analysis."
     },
     {
       "title": "Evaluating Sakana's AI Scientist: Bold Claims, Mixed Results, and a Promising Future?",
       "authors": ["Joeran Beel", "Min-Yen Kan"],
       "year": 2025,
       "arxiv_id": "2502.14297",
       "relevance": "Independent evaluation of The AI Scientist finding 42% experiment execution failure and methodological weaknesses."
     },
     {
       "title": "AI Scientists Fail Without Strong Implementation Capability",
       "authors": ["Minjun Zhu", "Qiujie Xie", "Yixuan Weng", "Jian Wu"],
       "year": 2025,
       "arxiv_id": "2506.01372",
       "relevance": "PaperBench evaluation showing 100% of agent-generated papers had methodological weaknesses; Claude 3.5 Sonnet achieved only 1.8% task completion."
     },
     {
       "title": "PROV-AGENT: Unified Provenance for Tracking AI Agent Interactions in Agentic Workflows",
       "authors": ["Renan Souza", "Amal Gueroudji"],
       "year": 2025,
       "relevance": "Adapts W3C provenance standards to track decision lineage in agentic workflows — directly related to auditability infrastructure."
     },
     {
       "title": "DeepResearch Bench: A Comprehensive Benchmark for Deep Research Agents",
       "authors": ["Mingxuan Du", "Benfeng Xu"],
       "year": 2025,
       "arxiv_id": "2506.11763",
       "relevance": "Benchmark for deep research agents measuring long-horizon research task performance."
     },
     {
       "title": "ResearchRubrics: Auditing the Citation Integrity of Deep Research Agents",
       "authors": ["Anjali Sharma", "Christopher Lin"],
       "year": 2025,
       "arxiv_id": "2511.07685",
       "relevance": "Directly audits citation integrity in deep research settings, closely related to provenance soundness."
     },
     {
       "title": "DeepTRACE: Auditing Deep Research AI Systems for Tracking Reliability Across Citations and Evidence",
       "authors": ["Pranav Narayanan Venkit", "Philippe Laban"],
       "year": 2026,
       "relevance": "Shows deep research agents generate one-sided answers with 40-80% citation accuracy — key empirical evidence for the auditability problem."
     },
     {
       "title": "Agentic Misalignment: How LLMs Could Be Insider Threats",
       "authors": ["Anthropic"],
       "year": 2025,
       "relevance": "Documents how AI agents bypass stated constraints to pursue alternative goals — relevant to objective drift analysis."
     },
     {
       "title": "AI models collapse when trained on recursively generated data",
       "authors": ["Ilia Shumailov", "Zakhar Shumaylov", "Yiren Zhao", "Yarin Gal", "Nicolas Papernot", "Ross Anderson"],
       "year": 2024,
       "doi": "10.1038/s41586-024-07566-y",
       "relevance": "Model collapse from recursive AI-generated training data — motivates why unauditable agent outputs contaminate the scientific pipeline."
     },
     {
       "title": "Protecting scientific integrity in an age of generative AI",
       "authors": ["Wulfram Blau", "Vinton G. Cerf"],
       "year": 2024,
       "doi": "10.1073/pnas.2407886121",
       "relevance": "PNAS perspective on scientific integrity threats from generative AI — closely related to this paper's motivation."
     }
   ]
 }