ai-research-survey

scan-v4.json (19931B)

---

 {
   "scan_version": 4,
   "paper_type": "survey",
   "paper": {
     "title": "The 2025 AI Agent Index: Documenting Technical and Safety Features of Deployed Agentic AI Systems",
     "authors": [
       "Leon Staufer",
       "Kevin Feng",
       "Kevin Wei",
       "Luke Bailey",
       "Yawen Duan",
       "Mick Yang",
       "A. Pinar Ozisik",
       "Stephen Casper",
       "Noam Kolt"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.17753",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about documenting 30 agents, finding transparency gaps, and most developers sharing little safety information are all supported by detailed findings in Sections 4 and 6.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes descriptive claims about the current state of agent documentation. No causal claims are made.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 explicitly bounds the scope: 'Our inclusion criteria favor the most significant agents, which may affect generalizability. Public interest metrics favor consumer products over enterprise deployments. Domain-specific agents are excluded.'",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper considers alternative explanations: Chinese agents' missing safety documentation 'may simply not be documented publicly' (Section 4.2). The discussion also notes that missing safety information could reflect internal practices not shared publicly rather than absence of safety work.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "This is a documentation survey of 30 AI agents. It makes no empirical measurements and claims no outcome — it catalogs publicly available information. No proxy-outcome gap exists because no measurement is used to represent a broader construct.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 'Limitations and Outlook' provides a dedicated multi-paragraph discussion of limitations.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses specific threats: 'Public interest metrics favor consumer products over enterprise deployments,' 'The Index relies exclusively on publicly available information, which may miss internal evaluations,' 'The Index relies on English and Chinese documentation and may miss information available in other languages.'",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 explicitly states scope boundaries: domain-specific agents excluded, only publicly available and deployable agents included, presents a snapshot as of December 31, 2025, internal company agents remain opaque.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This research was supported by the MATS Research program, which provided funding for L.S. and M.Y. through research stipends.'",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Cambridge, University of Washington, Harvard Law School, Stanford, Concordia AI, UPenn, MIT, Hebrew University of Jerusalem. No authors are affiliated with the companies whose agents are indexed.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "MATS (ML Alignment Theory Scholars) is an AI safety research program with no direct financial interest in the evaluation outcome of any specific AI agent.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 operationalizes 'agent' through four measurable criteria (autonomy, goal complexity, environmental interaction, generality) with specific thresholds; autonomy levels L1-L5 from Feng et al. are explicitly defined and applied throughout.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions are explicitly enumerated in Section 1: (1) Agent Index of 30 systems, (2) ecosystem-wide trends, (3) three illustrative case studies.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 situates the work against the 2024 AI Agent Index (Casper et al.), Foundation Model Transparency Index, Princeton Holistic Agentic Leaderboard, and prior documentation frameworks (model cards, system cards, datasheets), explicitly noting how this work expands upon and differs from each.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": false,
           "justification": "Candidate discovery relied on LLM-based queries (ChatGPT, Claude, Gemini with research mode) which are non-deterministic; the same prompts would not reliably produce the same 95-candidate list, making true reproduction impossible even though prompts are provided in Appendix B.5.",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.1 and Figure 2 specify three criteria categories with explicit thresholds: all four agency criteria required, at least one of three impact criteria required (with quantitative cutoffs: 10,000 searches or $1B valuation), and all three practicality criteria required.",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": false,
           "justification": "No PRISMA or equivalent structured review protocol is mentioned or followed; the paper uses a custom inclusion framework appropriate to an agent audit rather than a literature review.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": true,
           "justification": "Appendix B.5 provides the full LLM prompts used to discover candidate agents, Appendix C.1 provides the search term generation prompts and criteria, and Figure 15 shows the full list of search terms evaluated by Ahrefs.",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": true,
           "justification": "Sources listed include the 2024 AI Agent Index, Princeton Holistic Agent Leaderboard, AIAgentList.com, Ahrefs API for search volume, Crunchbase and Epoch AI for valuations, and stock exchanges for market capitalization.",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.3 documents the 95-candidate initial list, screening against inclusion criteria, ambiguous case handling, consultation with Chinese ecosystem experts, and cross-referencing; Appendix B.1 enumerates all 95 candidates with bold indicating inclusion.",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": true,
           "justification": "The paper explains the rationale for focusing on 'highly agentic systems with high-impact real-world applications,' contrasts with the broader 2024 Index, and justifies the December 31, 2025 cutoff and the three agent categories based on interaction paradigm differences.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": true,
           "justification": "The paper documents variation and tensions across agents: safety framework gaps between US and Chinese companies, the asymmetry between capability benchmarks and safety documentation, and the unresolved web conduct norms (robots.txt compliance vs. functionality).",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": false,
           "justification": "While the paper assesses agent transparency quality, there is no systematic quality rubric or risk-of-bias assessment for the documentation sources themselves (official blog posts, help pages, demos); 'None found' vs. 'None' is a presence/absence distinction, not a quality weighting.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 6.2 explicitly states 'The Index relies exclusively on publicly available information, which may miss internal evaluations or risk management practices,' and the ethical considerations note that 'None found' is distinguished from confirmed absence to avoid over-attributing transparency failures.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": true,
           "justification": "The paper provides systematic frequency counts across all 30 agents and 45 fields (e.g., '133/240 safety fields have no information,' '20/30 support MCP,' '3/30 have third-party testing'), with Figures 3, 6, 11, and 12 visualizing distribution patterns.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": true,
           "justification": "Section 6.1 ties all observations directly to the annotation data (e.g., the 'safety washing' observation is grounded in the 133/240 missing safety fields and the 25/30 agents disclosing no internal safety results), and future directions track the documented gaps.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "133 out of 240 safety-related annotation fields have no public information available.",
       "evidence": "Figure 6 and Section 4.6 document this systematically across all 30 agents; browser agents (67/104, 64%) and enterprise agents (25/40, 63%) have the most missing safety fields.",
       "supported": "strong"
     },
     {
       "claim": "Only 3/30 agents have documented third-party testing, and 25/30 disclose no internal safety results.",
       "evidence": "Section 4.6 states 'Third-party testing is documented for only 3/30 agents (Anthropic Claude, OpenAI ChatGPT, OpenAI Codex). 25/30 agents disclose no internal safety results.'",
       "supported": "strong"
     },
     {
       "claim": "Model Context Protocol (MCP) is the dominant interoperability standard, supported by 20/30 agents.",
       "evidence": "Section 4.3 and Figure 12 document MCP support counts per category; enterprise agents show 13/13 MCP support.",
       "supported": "strong"
     },
     {
       "claim": "23/30 agents are fully closed source, with only 7 open-sourcing their agent framework or harness.",
       "evidence": "Section 4.3 lists the 7 open-source agents by name (Alibaba MobileAgent, Browser Use, ByteDance Agent TARS, Google Gemini CLI, n8n Agents, OpenAI Codex, WRITER).",
       "supported": "strong"
     },
     {
       "claim": "Chinese-incorporated agent developers systematically lack publicly documented safety frameworks (1/5) and compliance standards (1/5) relative to US developers.",
       "evidence": "Section 4.2 and Figure 4b show 0% of Chinese companies (except Z.ai) publish AI safety frameworks, compared to 75% of US developers.",
       "supported": "strong"
     },
     {
       "claim": "Browser-based agents operate at the highest autonomy levels (L4-L5) with the most limited intervention mechanisms.",
       "evidence": "Section 4.4 and Figure 5 document browser agents at L4-L5; 'Browser Use's agent and Perplexity's Comet perform tasks autonomously once prompted, with no means for user involvement during execution.'",
       "supported": "strong"
     },
     {
       "claim": "Only 4 of 30 agents (ChatGPT Agent, OpenAI Codex, Claude Code, Gemini 2.5 Computer Use) provide agent-specific system cards.",
       "evidence": "Section 6.1 explicitly names these four, noting that others rely only on base model documentation.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "observational",
     "case-study",
     "qualitative"
   ],
   "key_findings": "The 2025 AI Agent Index documents 30 highly agentic deployed systems across 45 fields, finding pervasive transparency gaps: 133/240 safety-related fields had no public information, only 3/30 agents have third-party testing documentation, and only 4/30 provide agent-specific system cards. The ecosystem is heavily concentrated around three frontier model families (GPT, Claude, Gemini), creating systemic dependency risks. Browser-based agents operate at the highest autonomy levels (L4-L5) with the weakest safety documentation, while enterprise agent builders systematically delegate safety responsibility to end users. A transparency asymmetry exists where capability benchmarks are more frequently reported than safety evaluations, consistent with a weak form of 'safety washing.'",
   "red_flags": [
     {
       "flag": "Non-deterministic discovery",
       "detail": "Candidate agents were surfaced via LLM-based queries (ChatGPT, Claude, Gemini) which are inherently non-deterministic; the same prompts would not reliably reproduce the same 95-candidate pool, undermining reproducibility claims."
     },
     {
       "flag": "Low developer response rate",
       "detail": "Only 23% of contacted companies offered any response, and only 4/30 provided substantive corrections; many annotations may contain inaccuracies that were not verified by the companies being described."
     },
     {
       "flag": "Public-information-only constraint",
       "detail": "The entire index is based on publicly available documentation, which means transparency scores reflect communication quality rather than actual safety practices; companies with good marketing may outscore genuinely safer but less vocal companies."
     },
     {
       "flag": "No competing interests declaration",
       "detail": "No formal competing interests statement is provided for all authors; several authors are associated with institutions (MIT, Harvard, Stanford) that have active relationships with some of the evaluated companies."
     },
     {
       "flag": "LLM-assisted annotation verification",
       "detail": "GPT-5.2 was used to cross-check annotations for factual accuracy; this introduces potential systematic bias if the model has knowledge gaps or biases about specific agents, particularly less prominent ones."
     }
   ],
   "cited_papers": [
     {
       "title": "The AI Agent Index (2024)",
       "relevance": "Direct predecessor paper; the 2025 Index expands upon its methodology and inclusion criteria for comparative context."
     },
     {
       "title": "The 2024 Foundation Model Transparency Index",
       "relevance": "Comparable documentation effort for foundation models; used as a developer significance criterion and methodological comparison."
     },
     {
       "title": "Harms from Increasingly Agentic Algorithmic Systems",
       "relevance": "Characterizes AI agent properties (autonomy, goal complexity, environmental interaction) that form the theoretical basis for the Index's inclusion criteria."
     },
     {
       "title": "Levels of Autonomy for AI Agents",
       "relevance": "Provides the L1-L5 autonomy framework used throughout the Index to classify agent autonomy levels."
     },
     {
       "title": "Visibility into AI Agents",
       "relevance": "Related work on agent oversight mechanisms and transparency, directly relevant to the Index's ecosystem interaction category."
     },
     {
       "title": "Characterizing AI Agents for Alignment and Governance",
       "relevance": "Provides theoretical grounding for agent characterization used in the inclusion criteria alongside Chan et al."
     },
     {
       "title": "Holistic Agent Leaderboard: The Missing Infrastructure for AI Agent Evaluation",
       "relevance": "Concurrent work on agent evaluation infrastructure; cross-referenced during candidate identification."
     },
     {
       "title": "Safetywashing: Do AI Safety Benchmarks Actually Measure Safety Progress?",
       "relevance": "Provides the 'safety washing' framing used in Section 6.1 to characterize the transparency asymmetry between capability and safety documentation."
     },
     {
       "title": "Infrastructure for AI Agents",
       "relevance": "Directly related work on web conduct norms and agent identification protocols discussed in Section 4.5 and 6.1."
     },
     {
       "title": "Accountability in an Algorithmic Society",
       "relevance": "Provides the accountability diffusion framing used to analyze fragmented responsibility across the agent ecosystem."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Useful reference index for comparing AI agents but not a tool or technique practitioners can directly apply in their workflow."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The stark numbers — 25/30 agents disclose no safety results, only 4/30 have agent-specific system cards — and the 'safety washing' claim challenge the industry's safety narrative."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Safety transparency gaps are a central theme with concrete data showing browser agents at L4-L5 autonomy lacking evaluations, plus documented prompt injection incidents."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Directly names companies with missing safety documentation, coins 'safety washing', and highlights Perplexity's robots.txt evasion and Amazon's legal threats."
     },
     "demo_ability": {
       "score": 2,
       "justification": "The full index is browsable at aiagentindex.mit.edu with structured data downloadable from Zenodo in JSON and CSV formats."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "Covers ChatGPT, Claude, Gemini, and Copilot; authors from MIT, Cambridge, Harvard, and Stanford with an MIT-hosted website."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "47279778",
         "title": "Nested Training for Mutual Adaptation in Human-AI Teaming",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=47279778",
         "created_at": "2026-03-06T19:21:19Z"
       }
     ],
     "top_points": 2,
     "total_points": 2,
     "total_comments": 0
   }
 }