ai-research-survey

scan-v4.json (19428B)

---

 {
   "scan_version": 4,
   "paper_type": "position",
   "paper": {
     "title": "From Fluent to Verifiable: Claim-Level Auditability for Deep Research Agents",
     "authors": [
       "Razeen A Rasheed",
       "Somnath Banerjee",
       "Animesh Mukherjee",
       "Rima Hazra"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.13855",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are appropriately scoped as a 'perspective' paper. It proposes the AAR standard and identifies failure modes, both of which are developed in the body. No unsupported empirical claims in the abstract.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims (e.g., 'cosine similarity... making it mathematically incapable of representing entailment or contradiction' as a cause of citation decorrelation, Section 3.3) without controlled experiments. These are argued from first principles rather than demonstrated experimentally.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes broad claims about 'deep research agents' but its failure analysis draws primarily from evaluations of The AI Scientist and ChemCrow. The generalization from these specific systems to all deep research agents is not explicitly bounded.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 ('Alternative views and objections') addresses four counterarguments: bigger models will solve this, graphs are too expensive, logs are sufficient, and validation adds prohibitive latency. Each is engaged substantively.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical paper with no measurements. The proposed metrics (PCov, PSnd, CTran, AEff) are defined but not measured.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. Section 6 addresses objections but does not discuss limitations of the authors' own proposal (e.g., feasibility of building the proposed provenance graphs, scalability of the AAR metrics).",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats-to-validity discussion. The paper does not acknowledge that its failure taxonomy is derived from a small number of evaluated systems, nor that the proposed metrics are untested.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its framework does NOT address. It presents the AAR standard as broadly applicable without stating scope limitations.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Indian Institute of Science, IIT Kharagpur, Cisco Systems, TCG CREST.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed. One author is affiliated with Cisco Systems but there is no disclosure of whether this creates a conflict.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Core terms are formally defined: 'research-grade auditability' (Definition 1), 'provenance coverage' (Definition 2), 'provenance soundness' (Definition 3), 'contradiction transparency' (Definition 4), plus formal graph node types and edge relations (Definitions 5–9).",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The contribution is explicitly itemized in the introduction: (i) formalise requirements for auditable agents, (ii) propose a concrete provenance encoding, (iii) demonstrate practical instrumentation patterns. The reader knows what the paper claims to add.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The paper engages substantively with W3C PROV standards, AI Scientist evaluations, DeepTRACE, ResearchRubrics, PROV-AGENT, VeriLA, and HippoRAG — explaining how this work builds on and differs from each rather than merely listing references.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "position": {
       "argument_quality": {
         "argument_internally_consistent": {
           "applies": true,
           "answer": true,
           "justification": "The argument flows consistently: identify failure modes → formalize auditability → propose AAR metrics → argue for semantic provenance → rebut objections. No internal contradictions between premises and conclusions.",
           "source": "haiku"
         },
         "counterarguments_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Section 6 addresses four substantive counterarguments with specific rebuttals rather than dismissals, engaging with the strongest practical objections practitioners would raise.",
           "source": "haiku"
         },
         "analogies_appropriate": {
           "applies": true,
           "answer": true,
           "justification": "The analogy between human scientists (grounding claims in specific passages) vs. agents (opaque vector operations) is valid. The cosine similarity vs. logical entailment distinction is mathematically grounded, not a false equivalence.",
           "source": "haiku"
         },
         "prescriptions_proportional": {
           "applies": true,
           "answer": true,
           "justification": "Prescriptions (adopt AAR metrics, build semantic provenance graphs) are framed as design directions rather than mandates, proportional to the scope of documented failures. They are specific enough to be actionable without overclaiming certainty.",
           "source": "haiku"
         },
         "evidence_for_claims_cited": {
           "applies": true,
           "answer": true,
           "justification": "Factual claims are well-cited throughout: 44.2% failure rate from Cemri et al. [19], 42% experiment failure from Beel & Kan [32], 11,300+ retractions from Wiley/Hindawi [1,18], citation accuracy 40-80% from DeepTRACE [66].",
           "source": "haiku"
         },
         "alternatives_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 6 discusses three alternative approaches (model scaling, flat logs, post-hoc validation) and explains specifically why each cannot meet the four AAR properties.",
           "source": "haiku"
         },
         "historical_context_accurate": {
           "applies": true,
           "answer": true,
           "justification": "Historical references appear accurate: Popper's falsifiability criterion (1959) is correctly characterized, W3C PROV standard (2013) is correctly attributed, and Wiley/Hindawi retraction numbers are cited to primary news and journal sources.",
           "source": "haiku"
         }
       },
       "clarity_and_scope": {
         "key_terms_defined_precisely": {
           "applies": true,
           "answer": true,
           "justification": "Section 4 provides formal mathematical definitions for all four AAR metrics and all seven graph components. 'Auditability' is given a precise operational definition distinguishing it from mere logging.",
           "source": "haiku"
         },
         "engages_with_existing_literature": {
           "applies": true,
           "answer": true,
           "justification": "The paper engages substantively with DeepTRACE, ResearchRubrics, PROV-AGENT, and VeriLA, distinguishing how the AAR standard differs from existing provenance and evaluation frameworks rather than just listing them.",
           "source": "haiku"
         },
         "intended_audience_clear": {
           "applies": true,
           "answer": false,
           "justification": "The audience is never explicitly stated. The paper appears targeted at AI systems researchers and practitioners building deep research agents, but this must be inferred from context.",
           "source": "haiku"
         },
         "assumptions_stated": {
           "applies": true,
           "answer": false,
           "justification": "Key assumptions are not declared: that AI Scientist failure patterns generalize to all deep research agents; that NLI-based entailment scoring reliably operationalizes provenance soundness; that verification effort can practically be held below generation effort.",
           "source": "haiku"
         },
         "scope_of_applicability_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss where its argument does not apply — for instance, to narrow-scope retrieval agents, non-scientific AI writing systems, or deployment contexts where full auditability is infeasible.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "44.2% of failures in multi-agent systems arise from specification errors, task misinterpretation, and improper decomposition.",
       "evidence": "Cited from Cemri et al. (2025) analysis of multi-agent LLM system failure traces [19].",
       "supported": "moderate"
     },
     {
       "claim": "42% of AI Scientist proposed experiments fail to execute due to unresolved coding errors; PaperBench found 100% of agent-generated papers contained experimental weaknesses.",
       "evidence": "Cited from independent evaluations by Beel & Kan [32] and Zhu et al. [76] respectively.",
       "supported": "moderate"
     },
     {
       "claim": "Deep research agents exhibit citation accuracy of 40-80% across systems, with large fractions of unsupported statements.",
       "evidence": "Cited from DeepTRACE evaluation of deep research AI systems [66].",
       "supported": "moderate"
     },
     {
       "claim": "Current provenance standards record what agents did but fail to encode which sources substantiate each claim and how they support it.",
       "evidence": "Argued via comparative analysis of W3C PROV documentation and existing tooling (MLflow, DVC, PROV-AGENT) with citations to Moreau & Groth [48] and Souza et al. [62].",
       "supported": "moderate"
     },
     {
       "claim": "Wiley's Hindawi paper-mill investigations drove over 11,300 retractions by 2024.",
       "evidence": "Cited to ABC News (May 2024) and UKSG journal report [1, 18].",
       "supported": "strong"
     },
     {
       "claim": "The AAR standard's four metrics (PCov, PSnd, CTran, AEff) make auditability testable and comparable across systems.",
       "evidence": "Demonstrated only through a constructed conceptual example in Figure 4; no real system implementation, validation study, or inter-rater reliability data provided.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "theoretical",
     "case-study"
   ],
   "key_findings": "The paper argues that as AI-generated scientific reports become cheap to produce at scale, auditability rather than generation capability becomes the critical bottleneck for scientific trust. Three systematic failure modes in current deep research agents are identified and analyzed: objective drift (agents optimize for fluent output rather than truth preservation), transient constraints (specifications lost during long execution trajectories), and unverifiable inference chains (reasoning occurs through opaque vector operations leaving no traceable claim-evidence linkage). The AAR (Auditable Autonomous Research) standard is proposed with four measurable properties — provenance coverage, provenance soundness, contradiction transparency, and audit effort — operationalized through a formal semantic provenance graph schema. The paper's core prescription is that persistent, queryable semantic provenance graphs linking sources through typed reasoning nodes to claims are architecturally necessary for certifiable research agents, and that post-hoc verification cannot scale without this foundation built in during synthesis.",
   "red_flags": [
     {
       "flag": "Overclaims demonstration",
       "detail": "The contribution statement claims to 'demonstrate practical instrumentation that captures complete decision lineage at scale,' but the paper only provides a conceptual worked example (Figure 4) with constructed inputs. No real system was built, no real agent traces were instrumented, and no scale evaluation was conducted."
     },
     {
       "flag": "Single case study generalized",
       "detail": "Most concrete evidence for failure modes derives from a single system (The AI Scientist / Sakana AI), yet conclusions are stated as applying to 'current research agents' broadly without qualification."
     },
     {
       "flag": "No funding disclosed",
       "detail": "No funding source is disclosed anywhere in the paper despite institutional affiliations with IIT Kharagpur, Cisco Systems, and Indian Institute of Science."
     },
     {
       "flag": "No limitations section",
       "detail": "Section 6 rebuts external objections but does not acknowledge the paper's own limitations — e.g., that NLI entailment models may not reliably compute PSnd, that the AAR metrics have not been validated empirically, or that provenance graphs could themselves be manipulated."
     },
     {
       "flag": "Unstated assumptions",
       "detail": "Key assumptions driving the argument are not declared: that AI Scientist failure patterns generalize across agent architectures; that automated NLI scoring is reliable enough for PSnd measurement; and that verification effort can practically be kept below generation effort in real deployments."
     }
   ],
   "cited_papers": [
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "relevance": "Primary quantitative source for multi-agent failure rates (44.2% planning failures, 41-86.7% execution failures) used throughout to support the architectural failure taxonomy."
     },
     {
       "title": "The AI Scientist: Towards fully automated open-ended scientific discovery",
       "relevance": "Central case study system whose documented failures (energy efficiency paradox, novelty verification failures, cross-validation bugs) motivate the AAR standard."
     },
     {
       "title": "Evaluating Sakana's AI Scientist for Autonomous Research",
       "relevance": "Independent evaluation providing the energy-efficiency paradox example and 42% experiment failure rate that grounds the unverifiable inference chain discussion."
     },
     {
       "title": "PROV-AGENT: Unified Provenance for Tracking AI Agent Interactions in Agentic Workflows",
       "relevance": "Key prior provenance work for agentic systems that the paper builds on and distinguishes from by adding claim-level semantic links."
     },
     {
       "title": "ResearchRubrics: A Benchmark of Prompts and Rubrics For Evaluating Deep Research Agents",
       "relevance": "Related benchmark for evaluating citation integrity in deep research agents; positioned as complementary evidence for the auditability problem."
     },
     {
       "title": "DeepTRACE: Auditing Deep Research AI Systems for Tracking Reliability Across Citations and Evidence",
       "relevance": "Provides empirical citation accuracy data (40-80%) directly supporting synthesis failure claims; framed as reinforcing evidence for the AAR standard's necessity."
     },
     {
       "title": "AI models collapse when trained on recursively generated data",
       "relevance": "Nature paper on model collapse from AI-generated training data; cited to motivate urgency of auditability before agent outputs contaminate training pipelines."
     },
     {
       "title": "Lost in the Middle: How Language Models Use Long Contexts",
       "relevance": "Supports the transient memory failure mode — constraints specified early in long trajectories become inaccessible during later steps."
     },
     {
       "title": "AI Scientists Fail Without Strong Implementation Capability",
       "relevance": "PaperBench evaluation showing 100% of agent-generated papers contain weaknesses and Claude 3.5 Sonnet achieves only 1.8% task completion; strongest evidence that current agents are not research-grade."
     },
     {
       "title": "DeepResearch Bench: A Comprehensive Benchmark for Deep Research Agents",
       "relevance": "Recent benchmark defining long-horizon research evaluation tasks; provides context for what 'deep research' means in the scope of the paper's argument."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The AAR standard and formal provenance graph schema provide actionable design targets for practitioners building research agents, though no implementation or tooling is provided."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The framing that auditability rather than capability is now the field's primary bottleneck challenges the dominant benchmark-completion focus in agent research."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Raises concrete, documented concerns about AI-generated junk science contaminating Google Scholar and downstream training pipelines, with specific examples (Wiley retractions, NeurIPS hallucinated citations)."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "References Wiley/Hindawi retraction crisis and NeurIPS hallucinated citations, but the paper's tone is analytical rather than inflammatory."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Only a conceptual worked example (Figure 4) is provided; no system, tool, or code exists for practitioners to try or reproduce."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from IIT Kharagpur, Indian Institute of Science, and Cisco Systems are recognized institutions but the paper does not originate from a top-tier AI lab."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }