ai-research-survey

scan.json (26874B)

---

 {
   "paper": {
     "title": "Measuring Agents in Production",
     "authors": [
       "Melissa Z. Pan",
       "Negar Arabzadeh",
       "Riccardo Cogo",
       "Yuxuan Zhu",
       "Alexander Xiong",
       "Lakshya A Agrawal",
       "Huanzhi Mao",
       "Emma Shen",
       "Sid Pallerla",
       "Liana Patel",
       "Shu Liu",
       "Tianneng Shi",
       "Xiaoyuan Liu",
       "Jared Quincy Davis",
       "Emmanuele Lacavalla",
       "Alessandro Basile",
       "Shuyi Yang",
       "Paul Castro",
       "Daniel Kang",
       "Koushik Sen",
       "Dawn Song",
       "Joseph E. Gonzalez",
       "Ion Stoica",
       "Matei Zaharia",
       "Marquita Ellis"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.04123",
     "doi": "10.48550/arXiv.2512.04123"
   },
   "scan_version": 2,
   "active_modules": [],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code release is mentioned in the paper. The survey instrument and interview protocol are described but no code or data analysis scripts are released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Survey responses and interview data are described as anonymized and aggregated but no raw or processed dataset is released. No download link is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No computational environment details are provided. The paper uses Qualtrics for surveys and LOTUS for semantic aggregation but provides no environment specifications for reproducing the analysis."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While the survey questions are listed in Appendix G and interview protocol in Appendix C.3, there are no instructions for replicating the analysis pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 95% confidence intervals estimated from 1,000 bootstrap samples with replacement for categorical comparisons. Error bars are shown in figures (e.g., Figure 1)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports descriptive statistics with bootstrap confidence intervals but does not perform formal significance tests to compare groups or test hypotheses. Claims like '80% cite increased productivity' are presented as descriptive proportions without statistical testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports proportions and percentages but no formal effect sizes (e.g., Cohen's d, odds ratios). Differences between groups are presented as raw percentage differences without effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample sizes (20 interviews, 306 survey responses, 86 deployed systems) are reported but never justified. No power analysis or sample size rationale is given. The snowball sampling approach is described but without discussion of whether the achieved N is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Bootstrap-based confidence intervals serve as a variance measure for the survey proportions. The paper notes variation across responses in multiple figures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper positions itself as 'the first systematic study' of production agents but does not systematically compare its findings against prior surveys (e.g., LangChain 2024 survey of 1,300 professionals). Prior work is discussed in Section 2 but no quantitative comparison is made."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baseline comparisons are included, so contemporaneity cannot be assessed."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/interview study, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper examines multiple dimensions: 17 design dimensions across 4 research questions including models, architectures, prompting, evaluation methods, applications, and operational constraints."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of system outputs is not relevant here — this paper characterizes practices through surveys and interviews, not by evaluating a system."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a survey/interview study."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by deployment stage (deployed vs. all data in Appendix D), by domain (Figure 2), by organization maturity (Figure 15), and by many other dimensions throughout the paper."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses deployment challenges including reliability failures, evaluation difficulties, and security challenges. Case studies describe specific failure patterns (e.g., model upgrades breaking workflows)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: 75% evaluate without benchmarks, 26% have no meaningful baselines, agent deployments often fail or underdeliver (Section 1), and security mechanisms remain an open challenge."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (68% execute ≤10 steps, 70% rely on prompting, 74% depend on human evaluation) are all supported with specific data in Sections 4-7 with exact N values and percentages."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes several causal-adjacent claims, e.g., 'practitioners achieve reliability through system-level design rather than model-level or algorithmic advances' (Section 1) and 'teams deliberately choose simple, controllable methods... because they offer reliable agent performance.' These causal explanations are based on qualitative interview data, not controlled study design. The observational data cannot establish that simplicity *causes* reliability."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 explicitly bounds the study: 'Geographically, case study teams are concentrated in the Americas, with a few in Europe. Participation bias affects both data sources.' The paper states it documents 'the leading edge of production agents practice' rather than claiming comprehensive global coverage."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 discusses participation bias, temporal bias, geographic concentration, and self-selection effects. The paper acknowledges that respondents 'likely skew toward our professional networks' and that multi-month data collection may introduce temporal bias."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements — it reports what practitioners said in surveys and interviews without inflating these to broader claims. When practitioners report 'productivity gains,' the paper attributes this to practitioner self-report, not independently measured productivity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any AI model — it surveys practitioners. LOTUS is mentioned for semantic aggregation but is a tool used in analysis, not the subject of evaluation."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting as part of an experiment. LOTUS is used for domain normalization but this is an analysis tool, not the experimental method."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameters are relevant — this is a survey/interview study."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes survey design with 47 questions, piloting with external teams, dynamic branching in Qualtrics, and filtering to 86 deployed systems. Section B.1.2 details domain normalization with LOTUS and three independent annotators (Cohen's κ=0.636). The filtering pipeline from 306 responses to 86 deployed agents is documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 'Study Limitations' provides a dedicated discussion of scope and sampling limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 identifies specific threats: geographic concentration in the Americas, company policy filtering for interview acceptance, survey respondent skew toward professional networks, and temporal bias from multi-month data collection. These are specific to this study, not boilerplate."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 states: 'we view MAP as an initiative towards documenting the leading edge of production agents practice for open research rather than a comprehensive coverage of all agents development globally.' Geographic, participation, and temporal boundaries are explicitly stated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw survey data or interview transcripts are released. Only aggregated statistics are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection in detail: 30-90 minute semi-structured interviews with 2-5 researchers, 47-question survey in Qualtrics with dynamic branching, distribution channels (Berkeley RDI Summit, AI Alliance Meetup, MOOC, LinkedIn, Discord, X), collection period July 28 to October 29, 2025."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes snowball sampling starting from professional networks, iteratively broadening for diversity. Section 3.2 lists distribution channels. Section C.2.3 details participant recruitment through professional networks and agent-focused technical venues, with screening for active involvement."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from 306 survey responses to 86 deployed agents is documented (Section 3.2). Domain normalization with LOTUS and three annotators is described (Section B.1.2). Interview analysis using grounded theory open coding followed by focused coding is described (Section 3.1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section lists gifts from Accenture, Amazon, AMD, Anyscale, Broadcom Inc., Google, IBM, Intel, Intesa Sanpaolo, Lambda, Mibura Inc, Samsung SDS, and SAP."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: UC Berkeley, Intesa Sanpaolo, UIUC, Stanford University, IBM Research. The Intesa Sanpaolo affiliation is notable since the company is both an author institution and a funder."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Several funders (Amazon, Google, IBM, Intesa Sanpaolo) are companies that build and deploy AI agents. They have a financial interest in positive narratives about production agent adoption. Intesa Sanpaolo is both a funder and has co-authors on the paper. The study's findings about widespread agent adoption could benefit these companies commercially."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The Impact Statement mentions only data confidentiality, not author conflicts."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/interview study, not evaluating a pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no model evaluation on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no model evaluation on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration (OSF, AsPredicted, or equivalent). The study involved 306 survey respondents and 20 interview teams — human participants requiring pre-registration for scientific rigor."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned. The Impact Statement says 'All participant data was collected under confidentiality agreements' but does not mention institutional ethics review."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 14a reports participant roles (software/ML engineers 50.9%, technical executives 31.5%, etc.). Figure 14b reports deployment stages. Section C.2.1 describes organizational maturity levels and geographic distribution. However, individual demographics (gender, experience level, geography) of respondents are limited."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes filtering: responses filtered to 86 data points in production or pilot phases. For interviews, Section 3.1 describes iterative sample expansion guided by 'application diversity, organizational maturity, and global reach.' Section C.2.3 states participants were 'screened to ensure active, hands-on involvement.'"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational/cross-sectional survey study, not an experimental study with treatment conditions. Randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a cross-sectional survey/interview study. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 306 total valid responses filtered to 86 deployed systems. Individual question N values vary (e.g., N=66, N=69, N=53) showing different completion rates per question. The paper notes 'All questions were optional to respect participants' freedom to answer.'"
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/interview study, not proposing a computational method."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/interview study with negligible computational requirements."
       }
     }
   },
   "claims": [
     {
       "claim": "80% of practitioners deploy agents primarily to increase productivity",
       "evidence": "Figure 1, N=66, multi-select survey question. 80.3% (53/66) selected 'Increasing Productivity' among deployed agents.",
       "supported": "strong"
     },
     {
       "claim": "70% of deployed agents rely on prompting off-the-shelf models instead of weight tuning",
       "evidence": "Section 5.2, Figure 4: 14 of 20 case studies (70%) use off-the-shelf models without SFT or RL.",
       "supported": "moderate"
     },
     {
       "claim": "74% of deployed agents depend primarily on human-in-the-loop evaluation",
       "evidence": "Figure 7, N=31 deployed systems, 74.2% (23/31) report human-in-the-loop evaluation.",
       "supported": "strong"
     },
     {
       "claim": "68% of deployed agents execute at most 10 steps before human intervention",
       "evidence": "Figure 6a, N=60 deployed systems: 28 execute 1-4 steps and 13 execute 5-10 steps, totaling 68%.",
       "supported": "strong"
     },
     {
       "claim": "Production agents favor custom implementations (85%) over frameworks",
       "evidence": "Section 5.5: 17/20 case studies build custom in-house implementations. However survey data shows 61% use frameworks, creating tension with this finding.",
       "supported": "moderate"
     },
     {
       "claim": "Reliability is the top development challenge",
       "evidence": "Figure 11a, N=29: 37.9% rank Core Technical Performance (reliability, robustness, scalability) as top priority, exceeding all other categories.",
       "supported": "strong"
     },
     {
       "claim": "Practitioners achieve reliability through system-level design rather than algorithmic advances",
       "evidence": "Section 7.1 describes patterns: read-only agents, sandbox verification, internal deployment, wrapper APIs, role-based access. Interview data supports this but is qualitative and not independently verified.",
       "supported": "moderate"
     },
     {
       "claim": "75% of case study teams evaluate without benchmark sets",
       "evidence": "Section 6.1: 15 of 20 case studies (75%) evaluate without benchmarks, relying on A/B testing, user feedback, and production monitoring.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "qualitative",
     "observational"
   ],
   "key_findings": "MAP is the first large-scale study of production AI agents, combining 20 in-depth case studies and 306 practitioner surveys across 26 domains. Production agents favor simplicity and control: 70% use off-the-shelf models without fine-tuning, 68% execute ≤10 steps before human intervention, and 74% rely on human-in-the-loop evaluation. Reliability remains the top development challenge, addressed through systems-level design (read-only modes, sandboxing, constrained autonomy) rather than algorithmic advances. The study reveals a significant gap between research methods (emphasizing autonomy and RL) and production practices (favoring controllability and prompting).",
   "red_flags": [
     {
       "flag": "Recruitment bias",
       "detail": "Snowball sampling from UC Berkeley professional networks and specific events (Berkeley RDI Summit, AI Alliance Meetup) likely overrepresents Silicon Valley perspectives and organizations connected to the research group. The paper acknowledges this but the geographic and network concentration is substantial."
     },
     {
       "flag": "Non-independent funders",
       "detail": "Multiple funders (Amazon, Google, IBM, Intesa Sanpaolo) are major AI agent deployers with commercial interest in positive findings about agent adoption. Intesa Sanpaolo has both co-authors and funder status. No conflicts of interest statement is provided."
     },
     {
       "flag": "Small effective N for key claims",
       "detail": "Despite 306 total survey responses, after filtering to deployed agents many key findings rest on small N: N=31 for evaluation methods, N=29 for challenges, N=22 for number of models. Case study claims rest on N=20. Some percentages (e.g., 85% custom implementations) come from very small samples presented as strong findings."
     },
     {
       "flag": "No IRB/ethics review mentioned",
       "detail": "The study collected data from human participants through interviews and surveys without mentioning IRB or ethics board approval. This is a notable omission for a study published from a major research university."
     },
     {
       "flag": "Survivorship bias",
       "detail": "The study focuses on 'successful' production deployments by filtering to production/pilot systems. Failed or abandoned agent projects are largely excluded, potentially overrepresenting practices that appear successful due to selection."
     },
     {
       "flag": "Tension between interview and survey data",
       "detail": "Survey data shows 61% use frameworks, but case studies show 85% use custom implementations. The paper favors the case study finding in its narrative (Finding 10) without fully reconciling this contradiction."
     }
   ],
   "cited_papers": [
     {
       "title": "A survey on large language model based autonomous agents",
       "authors": ["Lei Wang", "Chen Ma", "Xueyang Feng"],
       "year": 2024,
       "relevance": "Foundational survey on LLM-based agent architectures and taxonomies, directly relevant to understanding the research landscape this paper characterizes."
     },
     {
       "title": "Survey on evaluation of LLM-based agents",
       "authors": ["Asaf Yehudai", "Lilach Eden"],
       "year": 2025,
       "arxiv_id": "2503.16416",
       "relevance": "Specialized survey on agent evaluation methods, complementary to MAP's empirical findings on evaluation practices in production."
     },
     {
       "title": "The emerged security and privacy of LLM agent: A survey with case studies",
       "authors": ["Feng He", "Tianqing Zhu"],
       "year": 2025,
       "doi": "10.1145/3773080",
       "relevance": "Survey on LLM agent security, relevant to this survey's findings that security is a secondary concern in current deployments."
     },
     {
       "title": "An illusion of progress? Assessing the current state of web agents",
       "authors": ["Tiange Xue"],
       "year": 2025,
       "arxiv_id": "2504.01382",
       "relevance": "Critical assessment of web agent capabilities that documents agent deployment failures, directly relevant to production-research gap identified by MAP."
     },
     {
       "title": "Why johnny can't use agents: Industry aspirations vs. user realities with AI agent software",
       "authors": ["Pranav Shome"],
       "year": 2025,
       "arxiv_id": "2509.14528",
       "relevance": "Studies capability gaps in agent software through user studies and marketing analysis, complementary industry perspective."
     },
     {
       "title": "DSPy: Compiling declarative language model calls into state-of-the-art pipelines",
       "authors": ["Omar Khattab"],
       "year": 2024,
       "relevance": "Prompt optimization framework mentioned as one of the rare automated prompting approaches used in production (9%)."
     },
     {
       "title": "ITBench: Evaluating AI agents across diverse real-world IT automation tasks",
       "authors": ["Saurabh Jha"],
       "year": 2025,
       "arxiv_id": "2502.05352",
       "relevance": "Benchmark for evaluating AI agents in IT automation, relevant to benchmark scarcity findings in production agent evaluation."
     },
     {
       "title": "The AI scientist: Towards fully automated open-ended scientific discovery",
       "authors": ["Cong Lu"],
       "year": 2024,
       "arxiv_id": "2408.06292",
       "relevance": "Agentic AI system for scientific discovery, represents research-side agent capabilities that contrast with production practices."
     },
     {
       "title": "Towards an AI co-scientist",
       "authors": ["Juraj Gottweis"],
       "year": 2025,
       "arxiv_id": "2502.18864",
       "relevance": "Single-system study of an AI agent for scientific discovery, exemplifies the case study literature MAP contrasts with."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Prominent coding agent framework, relevant to software engineering agent deployment patterns documented in MAP."
     },
     {
       "title": "Advances and challenges in foundation agents: From brain-inspired intelligence to evolutionary, collaborative, and safe systems",
       "authors": ["Bo Liu"],
       "year": 2025,
       "arxiv_id": "2504.01990",
       "relevance": "Comprehensive survey of agent architectures providing research-side perspective that MAP's production data contrasts with."
     },
     {
       "title": "Safety at scale: A comprehensive survey of large model and agent safety",
       "authors": ["Xiangyu Ma"],
       "year": 2025,
       "arxiv_id": "2502.05206",
       "relevance": "Survey on AI agent safety, relevant to MAP's finding that security is currently addressed through system-level constraints rather than dedicated safety mechanisms."
     },
     {
       "title": "LlamaFirewall: An open source guardrail system for building secure AI agents",
       "authors": ["Sridhar Chennabasappa"],
       "year": 2025,
       "arxiv_id": "2505.03574",
       "relevance": "Open-source security tool for AI agents, relevant to production agent security practices documented in MAP."
     }
   ]
 }