ai-research-survey

scan-v4.json (35988B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "Generative AI at Work",
     "authors": [
       "Erik Brynjolfsson",
       "Danielle Li",
       "Lindsey Raymond"
     ],
     "year": 2023,
     "venue": "Social Science Research Network",
     "arxiv_id": "2304.11771",
     "doi": "10.3386/w31161"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: 15% productivity increase (Table 2, Col 3: 0.301/1.97 = 15.2%), heterogeneity by skill (Figure 3), worker learning from outage analysis (Figure 6), gains largest for rare problems (Figure 7), customer sentiment improvement and reduced escalation (Figure 10, Table 4).",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses difference-in-differences with staggered rollout, includes a small RCT pilot (Section 4.1.1, Appendix Table A.1), instruments individual adoption with team-level adoption timing (Appendix Table A.2), tests parallel trends via event studies, and uses multiple robust DiD estimators. The causal identification strategy is well-justified for the main productivity claims.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds generalization: 'these findings apply for a particular AI tool, used in a single firm, within a single occupation, and should not be generalized across all occupations and AI systems' (Section 7). It also notes inability to observe wages, overall labor demand, or skill composition changes.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple alternative explanations are discussed: mean reversion (Section 4.2.1, Figure A.9), selection into treatment (addressed via agent FE and IV in Appendix Table A.2), selection on adherence vs. causal effect of following recommendations (Section 5.1), Hawthorne effects (implicitly via outage analysis), and confounds in attrition estimates (Section 6.3).",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully distinguishes its proxy measures from broader outcomes. RPH is presented as a specific industry metric, not a general productivity measure. The paper explicitly discusses what it cannot observe: 'we do not have access to pay data,' 'our paper is not designed to shed light on the aggregate employment or wage effects,' and discusses equilibrium effects it cannot measure (Section 7).",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The Conclusion (Section 7) contains extensive limitations discussion spanning multiple paragraphs covering single-firm generalization, equilibrium effects, wage data absence, potential ratchet effects on performance targets, and longer-run incentive challenges.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed throughout: manager selection bias in onboarding (addressed via IV, Section 4.1.2), mean reversion concern (Section 4.2.1, Figure A.9), inability to distinguish voluntary from involuntary attrition (Appendix A.3.5), lack of agent fixed effects for attrition (Section 6.3), and potential over-reliance on AI by top performers degrading future model quality.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 'our paper is not designed to shed light on the aggregate employment or wage effects' (Section 1), 'these findings apply for a particular AI tool, used in a single firm, within a single occupation' (Section 7), 'we do not have access to pay data' and cannot observe 'longer run equilibrium responses in worker demand or job design' (Section 7).",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the footnote: 'We thank... the Stanford Digital Economy Lab for funding. The content is solely the responsibility of the authors and does not necessarily represent the official views of Stanford University, MIT, or the NBER.'",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Brynjolfsson (Stanford & NBER), Li (MIT & NBER), Raymond (MIT). The paper refers to the AI firm and data firm anonymously but the authors are academic researchers, not employees of either firm.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Stanford Digital Economy Lab is an academic research center with no apparent financial stake in whether AI tools increase or decrease productivity. The funder appears independent of the outcome.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. While the authors appear to be independent academics, the absence of an explicit declaration is NO per the schema criteria.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 1 defines AI, ML, LLMs, and generative AI precisely; 'productivity' is operationalized through explicit metrics (RPH, AHT, CPH, resolution rate); 'adherence' is defined quantitatively in Section 5.1; the 'skill index' construction is detailed in Appendix A.2.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies itself as 'the first study of the impact of generative AI deployed at scale in the workplace,' with three explicit sets of findings on productivity, mechanisms, and worker experience of work.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The introduction engages substantively with the skill-biased technical change literature, prior AI adoption studies, and contemporaneous lab studies (Peng et al., Noy & Zhang, Dell'Acqua et al., Choi & Schwarcz), explicitly situating how this paper differs in real-world scope and duration.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No code repository or analysis scripts are provided or linked in the paper.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "The data comes from a Fortune 500 firm's proprietary customer service records. No data is released; this is understandable for confidentiality but still a NO.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No environment specifications, software versions, or computational setup details are provided for the econometric analysis.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are included. The econometric specifications are described (Equations 1-7, Appendix A.3) but no runnable code or scripts are provided.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "95% confidence intervals are shown on all event study figures (Figures 2, 3, 6, 8, 10, etc.) and robust standard errors are reported in parentheses in all regression tables.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": true,
           "justification": "Statistical significance is reported throughout with asterisks (*** p<0.01, ** p<0.05, * p<0.10) and clustered robust standard errors in all tables (Tables 2-4, A.1-A.11).",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Effect sizes are reported with baseline context throughout: e.g., '0.30 chats or 15.2%' off pre-treatment mean of 1.97 RPH (Table 2), '3.7 minute decrease... an 8.5% decline from the baseline mean of 43 minutes' (Table 3), customer sentiment improves by 0.18 points 'equivalent to half of a standard deviation' (Section 6.1).",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The sample size of 5,172 agents and 3 million chats is large and naturally determined by the firm's workforce, but no power analysis or formal sample size justification is provided.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Standard deviations are reported in summary statistics (Table 1, e.g., 'St. Average Handle Time 23-24 min'). Robust standard errors clustered at agent level are reported for all regressions. Multiple estimators are compared (Appendix Table A.9, Figure A.4).",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "The difference-in-differences design uses pre-treatment observations and never-treated agents as baselines. The paper also compares against multiple alternative DiD estimators (Sun-Abraham, Callaway-Sant'Anna, Borusyak et al., de Chaisemartin-D'Haultfœuille).",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "The robust DiD estimators used (Sun and Abraham 2021, Borusyak et al. 2022, Callaway and Sant'Anna 2021) represent the state of the art in causal inference for staggered adoption designs.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "The paper systematically decomposes productivity into components (AHT, CPH, resolution rate, NPS in Table 3), examines heterogeneity by skill quintile (Figure 3A), tenure (Figure 3B), adherence (Figure 5B), topic frequency (Figure 7), and examines outage periods to isolate learning from real-time assistance (Figure 6).",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Five main outcome metrics: resolutions per hour, average handle time, chats per hour, resolution rate, and net promoter score (Tables 2-3). Additional metrics include customer sentiment, agent sentiment, manager escalation rate, attrition, language fluency, and textual similarity.",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Human evaluation is used to validate LLM-generated topic classifications (3 independent human evaluators on 100 conversations, Appendix A.2.6) and language fluency scores (2 independent human reviewers on 100 conversations, Appendix A.2.5).",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": false,
           "answer": false,
           "justification": "This is an observational/quasi-experimental study, not a prediction task. The concept of held-out test sets does not apply to DiD causal inference designs.",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Extensive breakdowns by worker skill quintile (Figure 3A, A.6), tenure group (Figure 3B, A.7), adherence quintile (Figure 5B, A.12), topic frequency (Figure 7), agent location (Figure 8C-D), and adoption cohort (Figure A.10).",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The paper identifies where AI assistance fails or has negative effects: highest-skilled workers see 'small declines in quality' (Section 4.2.1, Figure A.6 Panels C-D showing negative effects on resolution rate and NPS for Q5 workers). The paper also discusses potential over-reliance by top performers.",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Several negative results: no significant impact on customer satisfaction (NPS) overall (Table 3 Col 4); negative quality effects for highest-skilled workers (Panels C-D of Figure A.6); top performers' over-reliance on AI may reduce future model quality (Section 5.1). The paper also notes that AI recommendations may 'distract top performers' (Section 4.2.1).",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper says the tool 'is built on a recent version of the Generative Pre-trained Transformer (GPT) family of large language models developed by OpenAI' (Section 2.2) but does not specify which GPT version (GPT-3, GPT-3.5, etc.) or any version identifier.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": false,
           "answer": false,
           "justification": "The AI tool is a deployed commercial product that generates real-time suggestions; the authors are studying its workplace impact, not designing or controlling prompts. The prompts/fine-tuning are internal to the AI firm.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "No hyperparameters for the AI model (temperature, sampling settings) or fine-tuning process are reported. The econometric specifications are well-documented but the AI system's technical parameters are not.",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "The paper evaluates a third-party commercial AI tool as deployed. The authors describe the tool's two main outputs (suggested responses and documentation links, Section 2.3, Appendix Figure A.1) but cannot be expected to describe internal scaffolding they have no access to.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.1 describes sample construction in detail: starting with 3,006,395 chats, dropping single-message chats, merging across databases via chat identifiers, winsorizing call duration at 99th percentile, and aggregating to agent-month level. Variable construction is detailed in Appendix A.2.",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Raw data is proprietary (Fortune 500 firm's customer service records) and not available for independent verification. Understandable given confidentiality constraints, but still NO.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Data collection is described in Appendix A.1: chat conversations from the firm's software systems (September 2019–June 2021), merged with internal company datasets and AI firm records. Agent information includes employer, location, manager/team, tenure, and AI onboarding date.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": true,
           "answer": true,
           "justification": "Agent selection for AI treatment is described: managers scheduled onboarding to minimize customer disruption, training sessions were limited by AI firm capacity, contractual license limits applied, and replacement occurred when AI-enabled agents left (Section 3.1). Agent employment details are in Section 2.2 and Appendix A.2.2.",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.1 documents the pipeline: starting sample (3M chats, 5,172 agents), merging steps using chat identifiers, dropping criteria (single-message chats, missing times/identifiers), winsorization (99th percentile), and aggregation to agent-month level. Topic classification (Appendix A.2.6) and sentiment scoring (A.2.4) pipelines are also documented.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "This study examines the impact of a deployed AI tool on worker productivity. It does not evaluate a pre-trained model's capability on any benchmark.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "Not a benchmark evaluation study. The paper studies workplace outcomes, not model capability on test sets.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "Not a benchmark evaluation study.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": true,
           "answer": false,
           "justification": "No pre-registration is mentioned for either the main observational study or the small RCT pilot. No link to OSF, AsPredicted, AEA registry, or similar.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": true,
           "answer": false,
           "justification": "No IRB or ethics board approval is mentioned anywhere in the paper, despite studying 5,172 workers whose chat data and performance records were analyzed.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": true,
           "answer": true,
           "justification": "Demographics are reported: 89% of agents are outside the US, mainly in the Philippines (Section 2.2, Table 1). Agent tenure distribution, employer type (direct vs. subcontractor), and geographic distribution across 25 locations are provided. Detailed breakdown in Table 1.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": true,
           "answer": true,
           "justification": "Inclusion criteria are described: agents providing chat-based technical support for US-based small businesses at the data firm and its subcontractors. Exclusions: chats with only one message, missing start/end times, missing identifiers (Appendix A.1). Agent sample construction is documented.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": true,
           "answer": true,
           "justification": "The staggered rollout mechanism is described in detail (Section 3.1): limited training session capacity, manager scheduling to minimize disruption, contractual license limits. The small RCT pilot is mentioned (Section 4.1.1, ~50 workers, half randomized to treatment), though limited detail on the randomization mechanism. The IV approach instruments individual adoption with team-level timing (Section 4.1.2).",
           "source": "opus"
         },
         "blinding_described": {
           "applies": true,
           "answer": false,
           "justification": "No blinding is described. Agents knew whether they had AI access (they received a 3-hour onboarding training). Whether managers or customers were blinded is not discussed.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": true,
           "answer": true,
           "justification": "Worker attrition is explicitly analyzed as an outcome (Section 6.3, Figure 11, Table A.11). The paper reports baseline attrition rate of 28.8% (Table A.11 DV Mean) and notes that attrition analysis drops pre-treatment observations for treated agents because they 'must survive to be treated' (Section 6.3, Appendix A.3.5).",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No costs of the AI system are reported. The paper mentions 'generative AI was costly and relatively untested' (Section 3.1) and that the firm had a 'limited budget for its deployment' but provides no specific cost figures.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No computational budget is stated for either the AI system or the econometric analysis.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Access to AI assistance increases worker productivity by 15% on average, as measured by issues resolved per hour.",
       "evidence": "DiD regression with agent and tenure fixed effects yields a coefficient of 0.301 RPH (Table 2, Column 3), 15.2% above the pre-treatment mean of 1.97 RPH; replicated across four robust DiD estimators (Table A.9).",
       "supported": "strong"
     },
     {
       "claim": "Less experienced and lower-skilled workers benefit disproportionately from AI assistance, with the lowest skill quintile seeing ~36% productivity gains versus near-zero for the highest quintile.",
       "evidence": "Figure 3A shows monotonically decreasing treatment effects from Q1 (0.527 RPH, ~36%) to Q5 (0.015 RPH, insignificant); Figure 3B shows analogous monotonic pattern by tenure, with <1-month agents gaining 0.707 RPH versus essentially zero for >12-month agents (Table A.6).",
       "supported": "strong"
     },
     {
       "claim": "AI assistance facilitates durable worker learning rather than mere reliance; workers retain productivity gains even when the AI system is unavailable during outages.",
       "evidence": "Figure 6 (Panel B vs. A) shows productivity improvements during outage periods grow with cumulative AI exposure time, unlike non-outage periods which show stable immediate gains; Panels C/D show this learning effect only occurs for high-adherence workers.",
       "supported": "moderate"
     },
     {
       "claim": "AI gains are largest for moderately rare customer problems where agents have less baseline experience but the AI has sufficient training data.",
       "evidence": "Figure 7A shows a non-monotonic pattern: largest gains (5–6 minute reduction) for 75th–90th percentile rarity topics vs. 4–5 minutes for most common and 4 minutes for rarest; Figure 7B shows a monotonic relationship when controlling for overall frequency, with least agent-familiar problems showing 15% vs. 10% reductions.",
       "supported": "strong"
     },
     {
       "claim": "AI assistance improves customer sentiment by half a standard deviation and reduces manager escalation requests by ~25%.",
       "evidence": "Table 4 shows customer sentiment coefficient of 0.177 (p<0.01, baseline mean=0.141, representing ~0.5 SD); Table 4 Column 3 shows -0.00875 escalation rate effect vs. baseline of 3.77%, approximately 23% reduction.",
       "supported": "strong"
     },
     {
       "claim": "AI assistance improves English language fluency particularly for Philippines-based agents.",
       "evidence": "Figure 8 shows significant event-study increases in comprehensibility and native fluency; Panels C/D show larger effects for Filipino agents, validated by human rater comparison showing LLM scores match human evaluators (p=0.22 and p=0.12 for no statistically significant difference).",
       "supported": "strong"
     },
     {
       "claim": "AI assistance reduces worker attrition by approximately 40% among newer agents.",
       "evidence": "Table A.11 shows -0.095 to -0.121 reduction in monthly attrition probability for agents with 0–2 months tenure, off a baseline of ~25%; the authors caveat that agent fixed effects cannot be included, so selection bias cannot be ruled out.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "observational",
     "rct"
   ],
   "key_findings": "A staggered quasi-natural experiment across 5,172 customer support agents finds that a GPT-based conversational AI assistant increases productivity (issues resolved per hour) by 15% on average, with substantially larger gains for novice and lower-skilled workers (~36% for lowest skill quintile vs. near-zero for highest) — inverting the traditional skill-biased technology complementarity pattern. Evidence from AI system outages demonstrates that gains partly reflect durable learning: workers who closely followed AI recommendations continued to outperform their pre-AI baseline even when the system was unavailable. Beyond productivity, AI assistance improved customer sentiment by half a standard deviation, reduced manager escalation requests by ~25%, decreased attrition among new workers by ~40%, and improved English fluency particularly for international agents.",
   "red_flags": [
     {
       "flag": "GPT version unspecified",
       "detail": "The AI tool is described only as 'built on a recent version of the GPT family'; no specific model version or training snapshot date is identified, making findings impossible to attribute to a specific system or replicate as AI capabilities evolve."
     },
     {
       "flag": "No IRB or ethics disclosure",
       "detail": "The study continuously monitored the communications and performance of 5,172 workers over 21 months without mentioning any IRB approval or informed consent process, raising ethical concerns given the workplace surveillance involved."
     },
     {
       "flag": "Proprietary data, no replication possible",
       "detail": "All data is proprietary to an unnamed private firm; neither raw data nor analysis code is released, making independent replication or verification of the results impossible."
     },
     {
       "flag": "No financial interests declaration",
       "detail": "No competing interests statement is provided; it is unclear whether authors have equity, consulting, or advisory relationships with OpenAI or the unnamed AI firm whose tool is the primary subject of study."
     },
     {
       "flag": "Attrition regression selection bias",
       "detail": "The attrition analysis cannot include agent fixed effects because attrition only occurs once per worker; the authors acknowledge this may overstate AI effects on retention if the firm preferentially assigned AI access to agents deemed more likely to stay."
     },
     {
       "flag": "Small RCT incomplete and underpowered",
       "detail": "The randomized pilot involved ~50 workers with control group members unidentifiable in the data (only treatment group of 22 can be recovered); main causal identification relies entirely on the quasi-experimental staggered rollout."
     }
   ],
   "cited_papers": [
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "relevance": "Peng et al. (2023) — key comparator lab study of AI coding assistant finding 2x speed improvement in a controlled task; this paper extends to real-world multi-month deployment with longitudinal learning effects"
     },
     {
       "title": "Experimental Evidence on the Productivity Effects of Generative Artificial Intelligence",
       "relevance": "Noy and Zhang (2023) — RCT showing ChatGPT improves professional writing task performance; companion study finding skill-compression consistent with this paper's heterogeneity results"
     },
     {
       "title": "Navigating the Jagged Technological Frontier: Field Experimental Evidence of the Effects of AI on Knowledge Worker Productivity and Quality",
       "relevance": "Dell'Acqua et al. (2023) — BCG consulting study finding GPT-4 helps within capability frontier but hurts outside it; directly analogous to this paper's finding that top performers can be harmed by AI"
     },
     {
       "title": "Estimating dynamic treatment effects in event studies with heterogeneous treatment effects",
       "relevance": "Sun and Abraham (2021) — primary causal identification strategy used throughout the paper to handle heterogeneous treatment effects in the staggered DiD design"
     },
     {
       "title": "When Are Combinations of Humans and AI Useful?",
       "relevance": "Vaccaro et al. (2024) — meta-analysis finding human-AI teams often underperform; this paper provides a more positive counterexample where augmentation works at scale"
     },
     {
       "title": "AI, Skill, and Productivity: The Case of Taxi Drivers",
       "relevance": "Kanazawa et al. (2022) — study of non-generative AI for taxi drivers finding low-skill drivers benefited most; consistent with this paper's central skill-heterogeneity finding"
     },
     {
       "title": "Robots and Jobs: Evidence from US Labor Markets",
       "relevance": "Acemoglu and Restrepo (2020) — canonical skill-biased automation paper; this paper explicitly positions against this prior work by showing generative AI inverts the typical complementarity pattern"
     },
     {
       "title": "More than a Feeling: Accuracy and Application of Sentiment Analysis",
       "relevance": "Hartmann et al. (2023) — provides the SiEBERT sentiment model used for customer and agent sentiment measurement, a core outcome variable"
     },
     {
       "title": "AI Assistance in Legal Analysis: An Empirical Study",
       "relevance": "Choi and Schwarcz (2023) — contemporaneous study of AI assistance for law students; part of the cluster of generative AI productivity studies this paper is situated within and contrasted against"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Direct evidence from a real enterprise deployment of an AI assistant across thousands of workers; immediately actionable for firms considering AI adoption decisions."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Counterintuitive finding that generative AI benefits lower-skilled workers most, inverting the traditional skill-biased technical change pattern; the durable learning result (gains persist during outages) also challenges the 'crutch' narrative."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Raises questions about labor displacement (12% fewer worker-hours to serve same volume) and AI training quality degradation as top workers stop innovating, but these are framed as open questions rather than immediate risks."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Modest conflict angle: top performers may be harmed by AI reducing their quality scores, and the incentive problem of top workers contributing to AI training data is raised as a concern."
     },
     "demo_ability": {
       "score": 2,
       "justification": "The general category of tools (ChatGPT, Copilot) is publicly accessible and relatable; the specific enterprise customer service system cannot be demoed but findings easily extrapolate to tools readers already use."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "Erik Brynjolfsson (Stanford/NBER) is one of the most prominent economists studying technology and labor; MIT/Stanford/NBER pedigree and Fortune 500 firm context add high credibility signal."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "43577957",
         "title": "A Study of Undefined Behavior Across Foreign Function Boundaries in Rust Libs",
         "points": 4,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43577957",
         "created_at": "2025-04-04T03:10:05Z"
       },
       {
         "hn_id": "36513194",
         "title": "On the Planning Abilities of Large Language Models – A Critical Investigation",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=36513194",
         "created_at": "2023-06-28T21:57:00Z"
       },
       {
         "hn_id": "31457199",
         "title": "Masked image modeling advances 3D medical image analysis",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=31457199",
         "created_at": "2022-05-21T12:15:54Z"
       },
       {
         "hn_id": "36053359",
         "title": "Why Is the Winner the Best?",
         "points": 1,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=36053359",
         "created_at": "2023-05-24T02:27:19Z"
       },
       {
         "hn_id": "44041341",
         "title": "Grounded in Context: Retrieval-Based Method for Hallucination Detection",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44041341",
         "created_at": "2025-05-20T13:23:42Z"
       },
       {
         "hn_id": "39736862",
         "title": "The Planning Abilities of LLMs: A Critical Investigation (2023)",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39736862",
         "created_at": "2024-03-17T18:45:17Z"
       },
       {
         "hn_id": "38586493",
         "title": "On the Planning Abilities of Large Language Models: A Critical Investigation",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=38586493",
         "created_at": "2023-12-09T21:57:24Z"
       },
       {
         "hn_id": "37627101",
         "title": "How Robust Is Google's Bard to Adversarial Image Attacks?",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37627101",
         "created_at": "2023-09-23T20:20:29Z"
       },
       {
         "hn_id": "37158226",
         "title": "What Types of Questions Require Conversation to Answer? AskReddit Study",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37158226",
         "created_at": "2023-08-17T07:07:27Z"
       },
       {
         "hn_id": "36105713",
         "title": "On the Planning Abilities of Large Language Models – A Critical Investigation",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=36105713",
         "created_at": "2023-05-28T16:51:29Z"
       }
     ],
     "top_points": 4,
     "total_points": 16,
     "total_comments": 2
   }
 }