ai-research-survey

scan.json (24194B)

---

 {
   "paper": {
     "title": "The 2025 AI Agent Index: Documenting Technical and Safety Features of Deployed Agentic AI Systems",
     "authors": [
       "Leon Staufer",
       "Kevin Feng",
       "Kevin Wei",
       "Luke Bailey",
       "Yawen Duan",
       "Mick Yang",
       "A. Pinar Ozisik",
       "Stephen Casper",
       "Noam Kolt"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.17753"
   },
   "scan_version": 3,
   "active_modules": [
     "survey_methodology"
   ],
   "methodology_tags": [
     "meta-analysis",
     "qualitative"
   ],
   "key_findings": "The 2025 AI Agent Index documents 30 deployed agentic AI systems across 1,350 annotation fields, finding significant transparency gaps especially around safety. Only 4/30 agents have agent-specific system cards, 25/30 disclose no internal safety results, and 23/30 have no third-party testing. The ecosystem concentrates on a few foundation models (GPT, Claude, Gemini), creating single points of failure. Browser agents operate at the highest autonomy levels (L4-L5) while often ignoring robots.txt and lacking safety evaluations.",
   "claims": [
     {
       "claim": "Most agent developers share little information about safety, evaluations, and societal impacts — 133/240 safety-related fields have no information available.",
       "evidence": "Section 4.6 and Figure 6 document missing safety fields across all 30 agents, broken down by agent category (browser 64%, enterprise 63%, chat 43%).",
       "supported": "strong"
     },
     {
       "claim": "Only 4/30 agents provide agent-specific system cards (ChatGPT Agent, OpenAI Codex, Claude Code, Gemini 2.5 Computer Use).",
       "evidence": "Section 4.6 explicitly lists these four agents as having dedicated agent-specific system cards.",
       "supported": "strong"
     },
     {
       "claim": "A transparency asymmetry exists suggesting a weaker form of 'safety washing' where high-level frameworks mask selective empirical disclosure.",
       "evidence": "Section 6.1 discusses this pattern but the characterization as 'safety washing' is interpretive rather than empirically demonstrated.",
       "supported": "moderate"
     },
     {
       "claim": "Browser-based agents operate at L4-L5 autonomy with limited mid-execution intervention.",
       "evidence": "Section 4.4 documents autonomy levels for all 30 agents with specific examples (Perplexity Comet, Browser Use).",
       "supported": "strong"
     },
     {
       "claim": "20/30 agents support MCP for tool integration.",
       "evidence": "Section 4.5 states this finding directly based on the annotation data.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No quality assessment of source documentation",
       "detail": "The Index documents what information is publicly available but does not assess the quality or accuracy of the documentation it catalogues. Safety frameworks are noted as present/absent without evaluating their substance."
     },
     {
       "flag": "Inclusion criteria favor well-funded entities",
       "detail": "The market significance criterion (≥$1B valuation) and public interest thresholds (10K searches, 20K GitHub stars) systematically exclude smaller or emerging agents. The authors acknowledge this in limitations."
     },
     {
       "flag": "LLM-assisted candidate discovery",
       "detail": "Initial candidate agents were surfaced using ChatGPT 5.2, Claude Sonnet 4.5, and Gemini 2.5 with research mode. While human experts made final decisions, this could bias toward agents these models know about."
     }
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The full annotations are released in JSON and CSV format on Zenodo (doi:10.5281/zenodo.18701931) and the Index is available at https://aiagentindex.mit.edu. The Generative AI Usage Statement mentions code for visualizations but does not mention releasing analysis code separately."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section A states 'The full annotations for all fields are available in JSON and CSV format on Zenodo at: https://doi.org/10.5281/zenodo.18701931'."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency information is provided. The paper mentions using Python for visualizations but provides no setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The annotation methodology is described in Section 3.4 and appendices, but there are no scripts or detailed procedures to reproduce the annotation process."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a documentation/survey paper reporting descriptive counts (e.g., 20/30 agents support MCP). No inferential statistics are used."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no comparative statistical claims requiring significance tests. All findings are descriptive counts and proportions."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No effect sizes are applicable — the paper reports descriptive proportions from a documentation exercise, not experimental comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 provides detailed inclusion criteria (agency, impact, practicality) that explain why 30 agents were selected from 95 candidates. The criteria are well-justified even if the resulting sample is not statistically representative."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs or repeated measurements are conducted. The paper documents factual properties of agents."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly compares against the inaugural 2024 AI Agent Index (Casper et al. [22]), noting revised inclusion criteria, expanded fields, and different response rates from developers."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 2024 AI Agent Index [22] is the most recent comparable work. The paper also references the Princeton Holistic Agentic Leaderboard [67] and AIAgentList.com [4] as concurrent efforts."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a documentation survey, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a documentation exercise, not an evaluation requiring metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of system outputs is not applicable to this documentation survey."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No test set is used — this is a survey/documentation paper."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are consistently broken down by agent category (chat, browser, enterprise) throughout Section 4 and in Figures 3, 5, 6, and 12."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses limitations of the Index methodology including coverage biases, reliance on public information, language limitations, and potential for missing qualifying systems."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative findings extensively: low developer response rates (23%), missing safety documentation (133/240 fields), lack of third-party testing (23/30), and the 'safety washing' pattern."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about documenting 30 agents, finding transparency gaps, and most developers sharing little safety information are all supported by detailed findings in Sections 4 and 6."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes descriptive claims about the current state of agent documentation. No causal claims are made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 explicitly bounds the scope: 'Our inclusion criteria favor the most significant agents, which may affect generalizability. Public interest metrics favor consumer products over enterprise deployments. Domain-specific agents are excluded.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper considers alternative explanations: Chinese agents' missing safety documentation 'may simply not be documented publicly' (Section 4.2). The discussion also notes that missing safety information could reflect internal practices not shared publicly rather than absence of safety work."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "This is a documentation survey of 30 AI agents. It makes no empirical measurements and claims no outcome — it catalogs publicly available information. No proxy-outcome gap exists because no measurement is used to represent a broader construct."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models are evaluated in experiments. The Generative AI Usage Statement mentions models used for auxiliary tasks but these are not experimental."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'detailed prompts in Section B.5' for the LLM-based candidate agent discovery process."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No model experiments are conducted requiring hyperparameter reporting."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in the methodology."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the full pipeline: LLM-based candidate surfacing (95 candidates) → screening against inclusion criteria → in-depth annotation → cross-validation → GPT-5.2 verification → developer review (4 weeks). Sections 3.1-3.4 and B.4-B.6 provide detailed criteria at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 'Limitations and Outlook' provides a dedicated multi-paragraph discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses specific threats: 'Public interest metrics favor consumer products over enterprise deployments,' 'The Index relies exclusively on publicly available information, which may miss internal evaluations,' 'The Index relies on English and Chinese documentation and may miss information available in other languages.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 explicitly states scope boundaries: domain-specific agents excluded, only publicly available and deployable agents included, presents a snapshot as of December 31, 2025, internal company agents remain opaque."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Full annotation data is released on Zenodo in JSON and CSV format (doi:10.5281/zenodo.18701931) and the Index is publicly available at https://aiagentindex.mit.edu."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 describes the data collection procedure in detail: annotation from public documentation, websites, demos, published papers, and governance documents. Seven subject-matter experts annotated according to category. All web sources were archived."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The data source is publicly available documentation about AI agents."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: 95 candidates surfaced via LLM queries → screened against inclusion criteria → annotated by 7 experts across categories → cross-validated with independent review → 37/1,350 discrepancies resolved → GPT-5.2 verification screening → 4-week developer review period."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This research was supported by the MATS Research program, which provided funding for L.S. and M.Y. through research stipends.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Cambridge, University of Washington, Harvard Law School, Stanford, Concordia AI, UPenn, MIT, Hebrew University of Jerusalem. No authors are affiliated with the companies whose agents are indexed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "MATS (ML Alignment Theory Scholars) is an AI safety research program with no direct financial interest in the evaluation outcome of any specific AI agent."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on any benchmark. It is a documentation survey of AI agents."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark evaluation of model capabilities is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper documents publicly available information about AI agents."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/documentation paper. Cost of the paper's own method is not applicable."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/documentation paper. No significant compute was required beyond standard research tools."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes a structured protocol: explicit inclusion criteria (agency, impact, practicality — Figure 2), systematic candidate identification via LLM queries and cross-referencing against multiple databases (Section 3.3), detailed annotation protocols developed through calibration exercises (Section B.4), and inter-annotator consistency measures."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The Index documents what information exists but does not assess the quality of the documentation it catalogues. Safety frameworks are recorded as present/absent without evaluating their rigor or substance."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses information availability bias: 'The Index relies exclusively on publicly available information, which may miss internal evaluations or risk management practices' (Section 6.2). It also distinguishes 'None found' from 'None' to address the bias of equating absence of public information with absence of practice."
       }
     }
   },
   "cited_papers": [
     {
       "title": "The AI Agent Index",
       "authors": [
         "Stephen Casper",
         "Luke Bailey",
         "Rosco Hunter",
         "Carson Ezell"
       ],
       "year": 2025,
       "arxiv_id": "2502.01635",
       "relevance": "Inaugural version of the AI Agent Index — directly comparable prior work on documenting deployed agentic AI systems."
     },
     {
       "title": "Harms from Increasingly Agentic Algorithmic Systems",
       "authors": [
         "Alan Chan",
         "Rebecca Salganik",
         "Alva Markelius"
       ],
       "year": 2023,
       "relevance": "Defines agency properties used in the Index's inclusion criteria and characterizes risks from agentic systems."
     },
     {
       "title": "Infrastructure for AI Agents",
       "authors": [
         "Alan Chan",
         "Kevin Wei",
         "Sihao Huang"
       ],
       "year": 2025,
       "relevance": "Discusses infrastructure and visibility challenges for AI agents, directly relevant to transparency and governance findings."
     },
     {
       "title": "AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents",
       "authors": [
         "Maksym Andriushchenko",
         "Alexandra Souly"
       ],
       "year": 2025,
       "relevance": "Benchmark for evaluating harmful capabilities of LLM agents, relevant to agent safety evaluation."
     },
     {
       "title": "Safetywashing: Do AI Safety Benchmarks Actually Measure Safety Progress?",
       "authors": [
         "Richard Ren",
         "Steven Basart",
         "Adam Khoja"
       ],
       "year": 2024,
       "relevance": "Directly relevant to the paper's finding about 'safety washing' patterns in agent documentation."
     },
     {
       "title": "Levels of Autonomy for AI Agents",
       "authors": [
         "K. J. Kevin Feng",
         "David W. McDonald",
         "Amy X. Zhang"
       ],
       "year": 2025,
       "relevance": "Provides the L1-L5 autonomy framework used throughout the Index to characterize agent autonomy levels."
     },
     {
       "title": "Security Challenges in AI Agent Deployment: Insights from a Large Scale Public Competition",
       "authors": [
         "Andy Zou",
         "Maxwell Lin",
         "Eliot Jones"
       ],
       "year": 2025,
       "relevance": "Large-scale evaluation of security challenges in AI agent deployment, relevant to agent safety benchmarking."
     },
     {
       "title": "OpenAgentSafety: A Comprehensive Framework for Evaluating Real-World AI Agent Safety",
       "authors": [
         "Sanidhya Vijayvargiya"
       ],
       "year": 2025,
       "relevance": "Framework for evaluating real-world AI agent safety, directly relevant to the Index's safety evaluation findings."
     },
     {
       "title": "The 2025 Foundation Model Transparency Index",
       "authors": [
         "Alexander Wan",
         "Kevin Klyman",
         "Sayash Kapoor"
       ],
       "year": 2025,
       "relevance": "Comparable transparency documentation effort for foundation models that the Index references for developer significance criteria."
     },
     {
       "title": "BrowseSafe: Understanding and Preventing Prompt Injection Within AI Browser Agents",
       "authors": [
         "Kaiyuan Zhang",
         "Mark Tenenholtz"
       ],
       "year": 2025,
       "relevance": "Addresses prompt injection in browser agents, directly relevant to the security incidents documented in the Index."
     },
     {
       "title": "Holistic Agent Leaderboard: The Missing Infrastructure for AI Agent Evaluation",
       "authors": [
         "Sayash Kapoor",
         "Benedikt Stroebl"
       ],
       "year": 2025,
       "relevance": "Concurrent work curating evaluations of agentic AI systems across benchmarks, complementary to the Index approach."
     },
     {
       "title": "Characterizing AI Agents for Alignment and Governance",
       "authors": [
         "Atoosa Kasirzadeh",
         "Iason Gabriel"
       ],
       "year": 2025,
       "relevance": "Provides characterization framework for AI agents used in the Index's agency inclusion criteria."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Useful reference index for comparing AI agents but not a tool or technique practitioners can directly apply in their workflow."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The stark numbers — 25/30 agents disclose no safety results, only 4/30 have agent-specific system cards — and the 'safety washing' claim challenge the industry's safety narrative."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Safety transparency gaps are a central theme with concrete data showing browser agents at L4-L5 autonomy lacking evaluations, plus documented prompt injection incidents."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Directly names companies with missing safety documentation, coins 'safety washing', and highlights Perplexity's robots.txt evasion and Amazon's legal threats."
     },
     "demo_ability": {
       "score": 2,
       "justification": "The full index is browsable at aiagentindex.mit.edu with structured data downloadable from Zenodo in JSON and CSV formats."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "Covers ChatGPT, Claude, Gemini, and Copilot; authors from MIT, Cambridge, Harvard, and Stanford with an MIT-hosted website."
     }
   }
 }