ai-research-survey

scan.json (28610B)

---

 {
   "paper": {
     "title": "Generative AI at Work",
     "authors": ["Erik Brynjolfsson", "Danielle Li", "Lindsey Raymond"],
     "year": 2023,
     "venue": "NBER Working Paper / arXiv",
     "arxiv_id": "2304.11771",
     "doi": "10.3386/w31161"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["observational", "rct"],
   "key_findings": "Access to a generative AI conversational assistant increases customer support worker productivity by 15% (resolutions per hour), with gains of ~30% for the least skilled/experienced workers and minimal impact on the most skilled. AI assistance facilitates durable worker learning, as shown by productivity gains persisting during system outages. Customer sentiment improves markedly (0.5 SD increase), manager escalation requests decline ~25%, and worker attrition decreases, especially among newer agents.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or analysis scripts are provided or linked in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The data comes from a Fortune 500 firm's proprietary customer service records. No data is released; this is understandable for confidentiality but still a NO."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, software versions, or computational setup details are provided for the econometric analysis."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. The econometric specifications are described (Equations 1-7, Appendix A.3) but no runnable code or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are shown on all event study figures (Figures 2, 3, 6, 8, 10, etc.) and robust standard errors are reported in parentheses in all regression tables."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Statistical significance is reported throughout with asterisks (*** p<0.01, ** p<0.05, * p<0.10) and clustered robust standard errors in all tables (Tables 2-4, A.1-A.11)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout: e.g., '0.30 chats or 15.2%' off pre-treatment mean of 1.97 RPH (Table 2), '3.7 minute decrease... an 8.5% decline from the baseline mean of 43 minutes' (Table 3), customer sentiment improves by 0.18 points 'equivalent to half of a standard deviation' (Section 6.1)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample size of 5,172 agents and 3 million chats is large and naturally determined by the firm's workforce, but no power analysis or formal sample size justification is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported in summary statistics (Table 1, e.g., 'St. Average Handle Time 23-24 min'). Robust standard errors clustered at agent level are reported for all regressions. Multiple estimators are compared (Appendix Table A.9, Figure A.4)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The difference-in-differences design uses pre-treatment observations and never-treated agents as baselines. The paper also compares against multiple alternative DiD estimators (Sun-Abraham, Callaway-Sant'Anna, Borusyak et al., de Chaisemartin-D'Haultfœuille)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The robust DiD estimators used (Sun and Abraham 2021, Borusyak et al. 2022, Callaway and Sant'Anna 2021) represent the state of the art in causal inference for staggered adoption designs."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically decomposes productivity into components (AHT, CPH, resolution rate, NPS in Table 3), examines heterogeneity by skill quintile (Figure 3A), tenure (Figure 3B), adherence (Figure 5B), topic frequency (Figure 7), and examines outage periods to isolate learning from real-time assistance (Figure 6)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five main outcome metrics: resolutions per hour, average handle time, chats per hour, resolution rate, and net promoter score (Tables 2-3). Additional metrics include customer sentiment, agent sentiment, manager escalation rate, attrition, language fluency, and textual similarity."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human evaluation is used to validate LLM-generated topic classifications (3 independent human evaluators on 100 conversations, Appendix A.2.6) and language fluency scores (2 independent human reviewers on 100 conversations, Appendix A.2.5)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational/quasi-experimental study, not a prediction task. The concept of held-out test sets does not apply to DiD causal inference designs."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns by worker skill quintile (Figure 3A, A.6), tenure group (Figure 3B, A.7), adherence quintile (Figure 5B, A.12), topic frequency (Figure 7), agent location (Figure 8C-D), and adoption cohort (Figure A.10)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies where AI assistance fails or has negative effects: highest-skilled workers see 'small declines in quality' (Section 4.2.1, Figure A.6 Panels C-D showing negative effects on resolution rate and NPS for Q5 workers). The paper also discusses potential over-reliance by top performers."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: no significant impact on customer satisfaction (NPS) overall (Table 3 Col 4); negative quality effects for highest-skilled workers (Panels C-D of Figure A.6); top performers' over-reliance on AI may reduce future model quality (Section 5.1). The paper also notes that AI recommendations may 'distract top performers' (Section 4.2.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: 15% productivity increase (Table 2, Col 3: 0.301/1.97 = 15.2%), heterogeneity by skill (Figure 3), worker learning from outage analysis (Figure 6), gains largest for rare problems (Figure 7), customer sentiment improvement and reduced escalation (Figure 10, Table 4)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses difference-in-differences with staggered rollout, includes a small RCT pilot (Section 4.1.1, Appendix Table A.1), instruments individual adoption with team-level adoption timing (Appendix Table A.2), tests parallel trends via event studies, and uses multiple robust DiD estimators. The causal identification strategy is well-justified for the main productivity claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds generalization: 'these findings apply for a particular AI tool, used in a single firm, within a single occupation, and should not be generalized across all occupations and AI systems' (Section 7). It also notes inability to observe wages, overall labor demand, or skill composition changes."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple alternative explanations are discussed: mean reversion (Section 4.2.1, Figure A.9), selection into treatment (addressed via agent FE and IV in Appendix Table A.2), selection on adherence vs. causal effect of following recommendations (Section 5.1), Hawthorne effects (implicitly via outage analysis), and confounds in attrition estimates (Section 6.3)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully distinguishes its proxy measures from broader outcomes. RPH is presented as a specific industry metric, not a general productivity measure. The paper explicitly discusses what it cannot observe: 'we do not have access to pay data,' 'our paper is not designed to shed light on the aggregate employment or wage effects,' and discusses equilibrium effects it cannot measure (Section 7)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says the tool 'is built on a recent version of the Generative Pre-trained Transformer (GPT) family of large language models developed by OpenAI' (Section 2.2) but does not specify which GPT version (GPT-3, GPT-3.5, etc.) or any version identifier."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The AI tool is a deployed commercial product that generates real-time suggestions; the authors are studying its workplace impact, not designing or controlling prompts. The prompts/fine-tuning are internal to the AI firm."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters for the AI model (temperature, sampling settings) or fine-tuning process are reported. The econometric specifications are well-documented but the AI system's technical parameters are not."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates a third-party commercial AI tool as deployed. The authors describe the tool's two main outputs (suggested responses and documentation links, Section 2.3, Appendix Figure A.1) but cannot be expected to describe internal scaffolding they have no access to."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 describes sample construction in detail: starting with 3,006,395 chats, dropping single-message chats, merging across databases via chat identifiers, winsorizing call duration at 99th percentile, and aggregating to agent-month level. Variable construction is detailed in Appendix A.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The Conclusion (Section 7) contains extensive limitations discussion spanning multiple paragraphs covering single-firm generalization, equilibrium effects, wage data absence, potential ratchet effects on performance targets, and longer-run incentive challenges."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed throughout: manager selection bias in onboarding (addressed via IV, Section 4.1.2), mean reversion concern (Section 4.2.1, Figure A.9), inability to distinguish voluntary from involuntary attrition (Appendix A.3.5), lack of agent fixed effects for attrition (Section 6.3), and potential over-reliance on AI by top performers degrading future model quality."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 'our paper is not designed to shed light on the aggregate employment or wage effects' (Section 1), 'these findings apply for a particular AI tool, used in a single firm, within a single occupation' (Section 7), 'we do not have access to pay data' and cannot observe 'longer run equilibrium responses in worker demand or job design' (Section 7)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data is proprietary (Fortune 500 firm's customer service records) and not available for independent verification. Understandable given confidentiality constraints, but still NO."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in Appendix A.1: chat conversations from the firm's software systems (September 2019–June 2021), merged with internal company datasets and AI firm records. Agent information includes employer, location, manager/team, tenure, and AI onboarding date."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Agent selection for AI treatment is described: managers scheduled onboarding to minimize customer disruption, training sessions were limited by AI firm capacity, contractual license limits applied, and replacement occurred when AI-enabled agents left (Section 3.1). Agent employment details are in Section 2.2 and Appendix A.2.2."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 documents the pipeline: starting sample (3M chats, 5,172 agents), merging steps using chat identifiers, dropping criteria (single-message chats, missing times/identifiers), winsorization (99th percentile), and aggregation to agent-month level. Topic classification (Appendix A.2.6) and sentiment scoring (A.2.4) pipelines are also documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the footnote: 'We thank... the Stanford Digital Economy Lab for funding. The content is solely the responsibility of the authors and does not necessarily represent the official views of Stanford University, MIT, or the NBER.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Brynjolfsson (Stanford & NBER), Li (MIT & NBER), Raymond (MIT). The paper refers to the AI firm and data firm anonymously but the authors are academic researchers, not employees of either firm."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Stanford Digital Economy Lab is an academic research center with no apparent financial stake in whether AI tools increase or decrease productivity. The funder appears independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. While the authors appear to be independent academics, the absence of an explicit declaration is NO per the schema criteria."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This study examines the impact of a deployed AI tool on worker productivity. It does not evaluate a pre-trained model's capability on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation study. The paper studies workplace outcomes, not model capability on test sets."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for either the main observational study or the small RCT pilot. No link to OSF, AsPredicted, AEA registry, or similar."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned anywhere in the paper, despite studying 5,172 workers whose chat data and performance records were analyzed."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Demographics are reported: 89% of agents are outside the US, mainly in the Philippines (Section 2.2, Table 1). Agent tenure distribution, employer type (direct vs. subcontractor), and geographic distribution across 25 locations are provided. Detailed breakdown in Table 1."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Inclusion criteria are described: agents providing chat-based technical support for US-based small businesses at the data firm and its subcontractors. Exclusions: chats with only one message, missing start/end times, missing identifiers (Appendix A.1). Agent sample construction is documented."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "The staggered rollout mechanism is described in detail (Section 3.1): limited training session capacity, manager scheduling to minimize disruption, contractual license limits. The small RCT pilot is mentioned (Section 4.1.1, ~50 workers, half randomized to treatment), though limited detail on the randomization mechanism. The IV approach instruments individual adoption with team-level timing (Section 4.1.2)."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described. Agents knew whether they had AI access (they received a 3-hour onboarding training). Whether managers or customers were blinded is not discussed."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Worker attrition is explicitly analyzed as an outcome (Section 6.3, Figure 11, Table A.11). The paper reports baseline attrition rate of 28.8% (Table A.11 DV Mean) and notes that attrition analysis drops pre-treatment observations for treated agents because they 'must survive to be treated' (Section 6.3, Appendix A.3.5)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No costs of the AI system are reported. The paper mentions 'generative AI was costly and relatively untested' (Section 3.1) and that the firm had a 'limited budget for its deployment' but provides no specific cost figures."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated for either the AI system or the econometric analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "Access to AI assistance increases worker productivity by 15% as measured by resolutions per hour.",
       "evidence": "Table 2 Column 3: coefficient 0.301 (p<0.01) with agent, year-month, and tenure FE, off pre-treatment mean of 1.97 RPH. Confirmed by Sun-Abraham event study (Figure 2) showing immediate and persistent effect.",
       "supported": "strong"
     },
     {
       "claim": "Less-skilled workers see productivity gains of ~36% while the most skilled workers see no significant gains.",
       "evidence": "Figure 3A: Q1 (lowest skill) gains 0.5 RPH (~36%), Q5 (highest skill) gains ~0 RPH. Table A.6 Column 1: Q1 coefficient 0.527 (p<0.01), Q5 coefficient 0.015 (not significant). Controlling for tenure FE.",
       "supported": "strong"
     },
     {
       "claim": "AI assistance leads to durable worker learning, not just real-time reliance on AI suggestions.",
       "evidence": "Figure 6: During AI system outages, workers with more prior AI exposure handle chats faster (Panel B shows increasing effect over months of exposure). Workers with high initial adherence show gains during outages (Panel C) while low-adherence workers do not (Panel D).",
       "supported": "moderate"
     },
     {
       "claim": "AI assistance improves customer sentiment by half a standard deviation.",
       "evidence": "Table 4 Column 1: customer sentiment improves by 0.177 points (p<0.01) on a -1 to 1 scale. Authors state this is 'equivalent to half of a standard deviation' (Section 6.1). Event study in Figure 10A shows immediate and persistent effect.",
       "supported": "strong"
     },
     {
       "claim": "Customer requests to speak to a manager decline by ~25% after AI deployment.",
       "evidence": "Figure 10C and Table 4 Column 3: coefficient -0.00875 (p<0.01) off baseline of 0.0377, approximately 23% decline. Event study shows gradual decline after AI introduction.",
       "supported": "strong"
     },
     {
       "claim": "Gains from AI adoption are largest for moderately rare problems, not the most common ones.",
       "evidence": "Figure 7A: largest duration reduction (5-6 min, 14%) for 75th-90th percentile topic rarity, vs. 4-5 min (10%) for most common topics. Figure 7B shows monotonic relationship when controlling for overall topic frequency: AI helps most for problems an individual agent encounters least often.",
       "supported": "moderate"
     },
     {
       "claim": "AI assistance improves English language fluency, especially for Philippines-based agents.",
       "evidence": "Figure 8: event studies show significant improvement in both comprehensibility and native fluency scores. Panels C-D show larger gains for Filipino agents than US-based agents. Validated against human evaluators (Appendix A.2.5).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product's impact",
       "detail": "While the academic authors are independent, the study relies on data from an anonymous 'AI firm' and 'data firm.' The AI firm tracks adherence and outage data, and the data firm selects which workers receive AI access. The relationship between the researchers and the firms is not fully disclosed — the AI firm provided the data and has a financial interest in positive results."
     },
     {
       "flag": "Non-random treatment assignment in main analysis",
       "detail": "The main analysis relies on staggered rollout where managers selected who got AI access and when. Though the paper addresses this with IV estimation and argues managers prioritized scheduling convenience, positive selection bias (giving AI to more promising workers first) cannot be fully ruled out. The RCT pilot (N=22 treated) is too small for robust inference."
     },
     {
       "flag": "No IRB or ethics approval mentioned",
       "detail": "The study analyzes detailed performance data, chat transcripts, and employment records of 5,172 workers without mentioning ethics review. While this may be covered by the firms' data use agreements, the absence of ethics oversight disclosure is notable for a study of workplace surveillance data."
     },
     {
       "flag": "Proprietary data prevents independent replication",
       "detail": "All data comes from anonymous firms and cannot be independently verified. No code, data, or reproduction materials are provided. This is a structural limitation but nonetheless means the results cannot be independently verified."
     }
   ],
   "cited_papers": [
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "relevance": "RCT studying Copilot's productivity impact on software developers — directly comparable methodology for AI-assisted coding productivity."
     },
     {
       "title": "Experimental Evidence on the Productivity Effects of Generative Artificial Intelligence",
       "authors": ["Shakked Noy", "Whitney Zhang"],
       "year": 2023,
       "relevance": "Online experiment showing ChatGPT access compresses productivity distribution for professional writing tasks, finding similar skill-heterogeneity patterns."
     },
     {
       "title": "Navigating the Jagged Technological Frontier: Field Experimental Evidence of the Effects of AI on Knowledge Worker Productivity and Quality",
       "authors": ["Fabrizio Dell'Acqua", "Edward McFowland III", "Ethan Mollick"],
       "year": 2023,
       "relevance": "Field experiment showing GPT-4 improves management consulting task quality within capabilities but hurts performance outside — directly relevant to AI-human complementarity."
     },
     {
       "title": "Do Users Write More Insecure Code with AI Assistants?",
       "authors": ["Neil Perry", "Megha Srivastava", "Deepak Kumar", "Dan Boneh"],
       "year": 2022,
       "relevance": "Study finding AI coding assistants may produce insecure code — relevant counterpoint to productivity gains narrative."
     },
     {
       "title": "GPTs are GPTs: An Early Look at the Labor Market Impact Potential of Large Language Models",
       "authors": ["Tyna Eloundou", "Sam Manning", "Pamela Mishkin", "Daniel Rock"],
       "year": 2023,
       "arxiv_id": "2303.10130",
       "relevance": "Taxonomy of LLM exposure across occupations — provides framework for understanding which jobs are most affected by AI tools."
     },
     {
       "title": "When Are Combinations of Humans and AI Useful?",
       "authors": ["Michelle Vaccaro", "Abdullah Almaatouq", "Thomas Malone"],
       "year": 2024,
       "relevance": "Meta-analysis of 100+ studies finding human-AI collaborations often underperform AI or humans alone — important counterpoint to this paper's positive findings."
     },
     {
       "title": "The Uneven Impact of Generative AI on Entrepreneurial Performance",
       "authors": ["Nicholas G. Otis", "Rowan Clarke", "Rembrand Koning"],
       "year": 2023,
       "relevance": "Study finding limited or negative effects of AI adoption in entrepreneurship — provides contrasting evidence on AI productivity impacts."
     },
     {
       "title": "The Adoption of ChatGPT",
       "authors": ["Anders Humlum", "Emilie Vestergaard"],
       "year": 2024,
       "relevance": "Study showing AI adoption depends on training access and regulations — relevant to understanding organizational factors in AI deployment."
     },
     {
       "title": "AI, Skill, and Productivity: The Case of Taxi Drivers",
       "authors": ["Kyogo Kanazawa", "Daiji Kawaguchi", "Hitoshi Shigeoka", "Yasutora Watanabe"],
       "year": 2022,
       "relevance": "Non-generative AI tool for taxi drivers showing 5% search time reduction with largest gains for low-skill workers — parallel finding in different domain."
     },
     {
       "title": "AI Assistance in Legal Analysis: An Empirical Study",
       "authors": ["Jonathan H. Choi", "Daniel Schwarcz"],
       "year": 2023,
       "relevance": "Empirical study of AI assistance for law students showing productivity compression, consistent with skill-heterogeneity findings."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "relevance": "Foundational work on LLM scaling laws — relevant to understanding the technical foundations of the AI tool studied."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang"],
       "year": 2022,
       "arxiv_id": "2203.02155",
       "relevance": "RLHF methodology used in the AI system's fine-tuning process — relevant to understanding how the tool was built."
     }
   ]
 }