ai-research-survey

scan-v5.json (19837B)

---

 {
   "scan_version": 5,
   "paper_type": "survey",
   "paper": {
     "title": "The 2025 AI Agent Index: Documenting Technical and Safety Features of Deployed Agentic AI Systems",
     "authors": [
       "Leon Staufer",
       "Kevin Feng",
       "Kevin Wei",
       "Luke Bailey",
       "Yawen Duan",
       "Mick Yang",
       "A. Pinar Ozisik",
       "Stephen Casper",
       "Noam Kolt"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.17753",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: the 30-agent index is delivered, transparency gaps are shown through Figure 3 (198/1350 fields 'None found'), and safety documentation absence is documented with 133/240 safety fields having no public information.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is primarily descriptive documentation; it makes no causal claims about what causes safety gaps or transparency differences.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly scopes claims to 30 publicly available general-purpose agents as of December 31, 2025, and Section 6.2 acknowledges this may not generalize to internal deployments or domain-specific agents.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly notes that Chinese companies lacking documented safety frameworks 'may simply not be documented publicly,' and consistently distinguishes 'None found' from 'None' to acknowledge that absence of public evidence is not evidence of absence.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently distinguishes between publicly available documentation (what is measured) and actual safety practices (what is claimed to be opaque), using the 'None found' vs 'None' distinction throughout.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 'Limitations and Outlook' is a dedicated section covering methodology, scope, generalizability, and language coverage.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats named include: inclusion criteria favoring significant agents (reducing generalizability), public-interest metrics favoring consumer over enterprise products, reliance solely on English and Chinese documentation, and exclusive use of publicly available information missing internal practices.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly excludes domain-specific agents, company-internal products, limited pre-releases, and agents requiring software engineering expertise to deploy; scope is bounded to 30 publicly available general-purpose agents as of December 31, 2025.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This research was supported by the MATS Research program, which provided funding for L.S. and M.Y. through research stipends.'",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are disclosed on the first page: Cambridge, UW, Harvard Law, Stanford, Concordia AI, UPenn, MIT (×2), and Hebrew University.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "MATS Research Program is an academic research fellowship program unrelated to the commercial AI agents being evaluated; no conflict between funder identity and the transparency findings reported.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no general competing interests or financial interests declaration for all authors; the only COI statement ('no conflicts of interest related to Anthropic or Claude Code') appears only in the Claude Code case study annotation, not as a paper-wide declaration.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 discusses the definition of 'agent' extensively drawing on prior literature; Section 3.1 operationalizes agency via four criteria (autonomy, goal complexity, environmental interaction, generality); autonomy levels L1-L5 are defined via Feng et al.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions are explicitly enumerated in Section 1: (1) the Agent Index of 30 systems across 45 fields, (2) ecosystem-wide trends, and (3) three case studies of distinct agent categories.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 engages substantively with the 2024 AI Agent Index (predecessor), the Princeton Holistic Agentic Leaderboard, Foundation Model Transparency Index, and a range of documentation frameworks (datasheets, model cards, system cards, factsheets), situating this work relative to each.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": false,
           "justification": "Initial candidate discovery used LLM-based queries (ChatGPT 5.2, Claude Sonnet 4.5, Gemini 2.5 with research mode); Section B.5 provides the prompts but LLM outputs are non-deterministic and a re-run would not produce the same 95-candidate list.",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": true,
           "justification": "Figure 2 and Section 3.1 specify explicit, operationalized inclusion criteria with quantitative thresholds: ≥10,000 searches or ≥20,000 GitHub stars, ≥$1B valuation, plus all-required agency and practicality sub-criteria.",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": false,
           "justification": "No mention of PRISMA or any other established systematic review protocol; the paper follows a custom inclusion framework.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": false,
           "justification": "Section C.1 provides LLM prompts used to generate search terms but does not enumerate the final list of actual search terms used per agent; terms were LLM-generated and not explicitly listed.",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": true,
           "justification": "Sources explicitly named: Ahrefs API (search volume), Google Scholar (paper counts), Yahoo Finance/Crunchbase/Epoch AI (market cap), GitHub (stars), and cross-references with 2024 AI Agent Index, Princeton Holistic Agent Leaderboard, and AIAgentList.com.",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.3 documents the pipeline: LLM queries surfaced 95 candidates → screened against inclusion criteria → ambiguous cases annotated in depth → final inclusion decisions → 30 agents included; key stage counts are provided.",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": true,
           "justification": "The paper explains the rationale for focusing on 'highly agentic systems with high-impact real-world applications' publicly available as of December 31, 2025, and explicitly justifies why domain-specific and internal agents are excluded.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": true,
           "justification": "The paper systematically acknowledges systematic differences across categories (frontier labs vs. enterprise platforms on safety evaluation, Chinese vs. US governance documentation patterns) and reports 37 inter-annotator discrepancies resolved through discussion.",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": false,
           "justification": "There is no formal quality rubric or risk-of-bias assessment for the agents or their source documentation; annotations distinguish 'None found' vs 'None' but do not score the reliability or rigor of what is documented.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 6.2 explicitly acknowledges that significance criteria 'favor well-funded companies and established products, potentially disadvantaging emerging developers and regional innovations,' and that exclusive reliance on public information may miss internal safety practices — the documentation-index equivalent of publication bias.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": true,
           "justification": "The paper provides systematic counts and percentages throughout: 20/30 support MCP, 15/30 reference safety frameworks, 133/240 safety fields missing, 23/30 fully closed source, 3/30 with third-party testing — constituting quantitative vote-counting synthesis.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": true,
           "justification": "Recommendations such as structured reporting requirements and evaluation targeting deployed tools rather than base models are directly grounded in documented gaps (133/240 safety fields missing, only 4 agent-specific system cards).",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Most developers share little information about safety, evaluations, and societal impacts: 133/240 safety-related fields have no public information.",
       "evidence": "Figure 6 and Section 4.6 document 133/240 safety fields across 30 agents as 'None found'; browser (64%) and enterprise (63%) agents have the highest missing rates.",
       "supported": "strong"
     },
     {
       "claim": "Only 3/30 agents have documented third-party testing; 25/30 disclose no internal safety results.",
       "evidence": "Section 4.6: 'Third-party testing is documented for only 3/30 agents (Anthropic Claude, OpenAI ChatGPT, OpenAI Codex). 25/30 agents disclose no internal safety results.'",
       "supported": "strong"
     },
     {
       "claim": "Chinese-developed agents have substantially lower safety documentation than US agents: 1/5 with safety frameworks vs. ~75% of US agents.",
       "evidence": "Figure 4b shows Chinese companies at 20% for AI Safety Framework and 20% for Compliance Standards, compared to 76% and 95% for US companies.",
       "supported": "strong"
     },
     {
       "claim": "Model Context Protocol has become the dominant interoperability standard, supported by 20/30 agents including all 13 enterprise platforms.",
       "evidence": "Section 4.5 and Figure 12: '20/30 agents explicitly support MCP'; all 13 enterprise agents support MCP versus 4/12 chat agents.",
       "supported": "strong"
     },
     {
       "claim": "Most agents (21/30) do not disclose their AI nature to end users or third parties by default, and only 3/30 support media watermarking.",
       "evidence": "Section 4.5: '21/30 agents have no documented default disclosure behavior. Only 3/30 agents support watermarking generated media (e.g., through SynthID and C2PA).'",
       "supported": "strong"
     },
     {
       "claim": "Browser-based agents frequently bypass robots.txt and anti-bot measures, with some explicitly marketing this capability.",
       "evidence": "Section 4.5 and 5.2: BrowserUse 'explicitly markets bypassing anti-bot systems'; Cloudflare documented Perplexity using undeclared Chrome-signature crawlers; only 6/30 agents explicitly state robots.txt compliance.",
       "supported": "strong"
     },
     {
       "claim": "The ecosystem exhibits concentrated model dependency: almost all non-frontier-lab agents rely on GPT, Claude, or Gemini model families.",
       "evidence": "Section 4.3 and 6.1: 'Only frontier labs themselves (Anthropic, Google, OpenAI) and Chinese developers run their own proprietary models; the majority rely primarily on GPT, Claude, or Gemini model families.'",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "observational",
     "case-study",
     "qualitative"
   ],
   "key_findings": "The 2025 AI Agent Index documents 45 fields across 30 deployed AI agents and finds widespread transparency gaps: 133/240 safety-related fields have no public information, only 3/30 agents have third-party testing documentation, and only 4/30 have agent-specific system cards. Chinese developers show markedly lower safety documentation than US counterparts (1/5 vs. ~75% with safety frameworks). The ecosystem is structurally concentrated around GPT/Claude/Gemini models, creating shared dependency risks. Browser agents operating at L4–L5 autonomy present the highest risk profile with the least safety documentation, and most agents bypass web conduct standards like robots.txt with active justifications from developers.",
   "red_flags": [
     {
       "flag": "Non-reproducible search",
       "detail": "Initial candidate discovery used LLM-based queries (ChatGPT 5.2, Claude Sonnet 4.5, Gemini 2.5) whose outputs are non-deterministic; a re-run with the same prompts would yield a different candidate list, undermining reproducibility of the 95-agent starting pool."
     },
     {
       "flag": "Low developer response rate",
       "detail": "Only 23% of developers offered any response to annotation review requests, and only 4/30 provided substantive corrections; findings likely substantially undercount internal safety practices that exist but are not published."
     },
     {
       "flag": "Conflates non-disclosure with absence",
       "detail": "While the paper distinguishes 'None found' from 'None,' the headline finding that '133/240 safety fields have no information' conflates genuine absence of safety measures with non-disclosure of existing ones, which the paper's own methodology acknowledges is ambiguous."
     },
     {
       "flag": "Small and biased sample",
       "detail": "30 agents selected by significance criteria that favor consumer-facing products and US/China companies; excludes domain-specific agents, internal deployments, and less prominent developers, making ecosystem-wide conclusions speculative."
     }
   ],
   "cited_papers": [
     {
       "title": "The AI Agent Index (Casper et al., 2025)",
       "relevance": "Predecessor 2024 index that this work directly updates and expands, including revised inclusion criteria and annotation fields"
     },
     {
       "title": "Visibility into AI Agents (Chan et al., 2024)",
       "relevance": "Prior work on transparency and documentation of AI agent systems, foundational to the Index's motivation and framing"
     },
     {
       "title": "The 2024 Foundation Model Transparency Index (Bommasani et al., 2024)",
       "relevance": "Comparable documentation effort for foundation models; used as an inclusion criterion for developer significance"
     },
     {
       "title": "Levels of Autonomy for AI Agents (Feng et al., 2025)",
       "relevance": "Provides the L1–L5 autonomy framework used throughout the Index to characterize and compare agent autonomy levels"
     },
     {
       "title": "Harms from Increasingly Agentic Algorithmic Systems (Chan et al., 2023)",
       "relevance": "Defines the agency criteria (autonomy, goal complexity, environmental interaction, generality) directly adopted for inclusion criteria"
     },
     {
       "title": "Holistic Agent Leaderboard (Kapoor et al., 2025)",
       "relevance": "Concurrent work documenting agentic AI systems across capability benchmarks; used for cross-referencing agent candidates"
     },
     {
       "title": "Safetywashing: Do AI Safety Benchmarks Actually Measure Safety Progress? (Ren et al., 2024)",
       "relevance": "Provides theoretical grounding for the 'safety-washing' concept applied to the transparency asymmetry observed between capability benchmarks and safety documentation"
     },
     {
       "title": "Infrastructure for AI Agents (Chan et al., 2025)",
       "relevance": "Discusses governance challenges for web-interacting agents, directly relevant to the robots.txt and web conduct findings"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly usable by policymakers, procurement teams, and researchers to compare agents on safety and transparency; the online Index at aiagentindex.mit.edu is a live, downloadable resource."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that 133/240 safety fields have no public information and only 3/30 agents have third-party testing challenges industry safety-responsibility narratives, though the general direction is expected."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Concretely documents L4–L5 autonomy browser agents with prompt injection vulnerabilities, agents designed to bypass anti-bot systems, and absence of safety oversight for most deployed systems."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Documents real legal disputes (Amazon vs. Perplexity, NYT vs. OpenAI, Reddit vs. Anthropic) and specific named prompt injection incidents against Perplexity Comet and Opera Neon."
     },
     "demo_ability": {
       "score": 3,
       "justification": "The full Index is publicly available at aiagentindex.mit.edu in JSON and CSV formats, and all 30 documented agents are themselves publicly accessible products users can try."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "Covers Anthropic Claude, OpenAI ChatGPT, Google Gemini, Microsoft Copilot Studio, Salesforce Agentforce, and 25 other flagship products from the most recognized AI companies."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "47279778",
         "title": "Nested Training for Mutual Adaptation in Human-AI Teaming",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=47279778",
         "created_at": "2026-03-06T19:21:19Z"
       }
     ],
     "top_points": 2,
     "total_points": 2,
     "total_comments": 0
   }
 }