ai-research-survey

scan.json (23542B)

---

 {
   "paper": {
     "title": "Scaling Laws for Economic Productivity: Experimental Evidence in LLM-Assisted Translation",
     "authors": ["Ali Merali"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2409.02391"
   },
   "scan_version": 2,
   "active_modules": [],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or mention of code release found anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset download link or data repository is mentioned. The experimental data from the 300 translators is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, software versions, or dependency lists are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions or scripts are provided. The experimental design is described but there are no step-by-step instructions for replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard errors are reported in parentheses for all regression coefficients in Tables 1-5 in Appendix A."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "P-values are reported for main results (e.g., p=0.001 for time reduction, p=0.000 for quality improvement, p=0.017 for skill heterogeneity). Regression tables use significance stars (*** p<0.01, ** p<0.05, * p<0.1)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with context: 12.3% quicker per 10x compute, 0.18 standard deviations grade improvement, 16.1% earnings increase per 10x compute, and 0.14 SD for any-AI quality increase."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample size of 300 translators and 1,800 tasks is stated but no power analysis or justification for why 300 participants was sufficient is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard errors are reported for all regression coefficients. R-squared and adjusted R-squared values are provided for all models."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "A control group with no AI assistance serves as the baseline, with results compared in Section 3.1 and Table 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 13 LLMs of varying compute sizes including models up to Claude 3.5 Sonnet (contemporary at time of writing) against a no-AI control."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an RCT measuring the effect of model compute on productivity, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: time taken, quality grade (7-point scale), and earnings per minute (inclusive of bonuses). These are analyzed separately in Sections 3.2-3.4."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Three experienced professional translators (5+ years experience each) graded each task on a 7-point scale, with incentives for consistent grading."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an RCT, not a model evaluation on a dataset split. There is no train/test split to assess."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by skill level (high-skill vs low-skill in Section 3.5), by language (Arabic, Hindi, Spanish in regression tables), and by task (tasks 2-5 in regression controls)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of failure cases, tasks where AI assistance hurt performance, or individual translator difficulties. Only aggregate positive trends are shown."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "The quality improvement from any AI usage was not statistically significant (p=0.148) which is reported, but no other negative results or failed approaches are discussed. The non-significant quality finding is somewhat buried rather than highlighted."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (12.3% speed improvement, 0.18 SD grade improvement, 16.1% earnings increase per 10x compute, 4x larger gains for lower-skilled) are all supported by regression results in Section 3 and Appendix A."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "This is a pre-registered RCT with random assignment to treatment conditions (different LLMs or control). The study design is adequate for the causal claims made about model compute effects on productivity."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only translation tasks but the abstract and Section 4 extrapolate to 'U.S. productivity' and 'aggregate productivity gains from AI over the next decade.' The title says 'Economic Productivity' broadly. Section 5 acknowledges limitations to translation and short tasks, but the 6.9% aggregate estimate in Section 4 extends well beyond translation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of confounds such as: participants might use AI differently based on familiarity, the short task format may not reflect real professional translation, learning effects across the 5 tasks, or whether productivity gains reflect genuine quality or surface-level fluency. Section 5 mentions limitations but does not discuss alternative explanations for the observed scaling relationship."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures task completion time, a 7-point grade from 3 evaluators, and earnings per minute, then frames these as 'productivity' and 'economic scaling laws.' No discussion of whether short online translation tasks proxy real professional translation productivity, or whether earnings per minute in an experimental setting maps onto actual economic productivity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 13 LLMs including 'Claude-3.5 Sonnet' and references GPT-2/GPT-4 but does not provide specific version IDs or API snapshot dates for the 13 models used. The models are referenced primarily by their training compute, not by exact versions."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Participants used LLMs as translation aids in a free-form manner; the experiment did not use specific prompts as part of the methodology. The participants chose how to interact with the models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, etc.) are reported for the 13 LLMs used in the experiment."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Participants interact directly with LLMs for translation assistance."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe data cleaning or preprocessing steps between raw experimental data and the regression analyses. No mention of how outliers were handled or how the final sample of 1,392-1,500 observations was derived from 300 participants × 6 tasks = 1,800 potential observations."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Discussion) includes explicit limitations: 'There are many limitations to this study. This paper focused on a single professional skill (translation) and only tested participants on short tasks. Further, the results were only derived on a range of just over two orders of magnitude of compute.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: single skill (translation), short tasks only, limited compute range (two orders of magnitude), and uncertainty about generalization to other domains. These are not generic boilerplate."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Section 5 mentions limitations, the paper does not explicitly state what the results do NOT show. The 6.9% aggregate productivity estimate in Section 4 extends well beyond what the data supports, and the scope boundaries around this extrapolation are not clearly drawn."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is made available. The individual translator responses, timing data, grades, and model interactions are not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the experimental design: 300 translators recruited via Freelancer and Fiverr, completing 6 tasks each (1 baseline + 5 experimental), with detailed task descriptions in Appendix B, grading by 3 professionals, and payment structure."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 states participants were recruited through Freelancer and Fiverr, split evenly across Spanish/Hindi/Arabic, with screening criteria (1+ year experience, paid tasks in past year, language standards, consent to monitoring)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "There is no documentation of the pipeline from raw experimental data to final regression tables. The observation counts vary between tables (1,500 vs 1,392 vs 1,391) with no explanation of why observations were dropped at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'I gratefully acknowledge financial support from Open Philanthropy to conduct this research.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation is disclosed as Yale University, Department of Economics. The paper does not evaluate a product made by Yale or Open Philanthropy."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Open Philanthropy is a major funder of AI safety and AI-related research with a known position that AI will be transformative. The paper's finding that AI scaling produces large productivity gains aligns with the funder's worldview. While not a direct financial conflict, the funder is not fully independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. There is no declaration of whether the author has any financial interests related to the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate pre-trained model capability on a benchmark. It is an RCT measuring human productivity when assisted by LLMs. Contamination of LLM training data is not relevant to the experimental design."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation. The paper measures human performance on novel translation tasks, not model performance on a test set."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation. The tasks are original translation passages, not drawn from a pre-existing benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": true,
         "justification": "The paper states it 'was preregistered at the AEA RCT Registry (AEARCTR-0013743)' in the abstract footnote."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'The research described in this article was approved by the Yale Human Research Protection Program.'"
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 reports: 70%+ had 3+ years experience, ~33% had 5+ years, self-reported translation ability 4.52/5, professional fluency 4.85/5, AI familiarity 4.15/5, AI ability 4.18/5. Languages split evenly across Spanish, Hindi, Arabic."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 states: at least one year professional experience, completed paid translation tasks in past year, met language ability standards, comfortable with AI usage monitoring. Baseline task served as quality screen."
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states participants were 'randomly assigned to either treatment groups where they could utilize one of thirteen LLMs... or to a control group' but does not describe the randomization procedure (stratification, tool used, block randomization, etc.)."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No mention of blinding. It is unclear whether participants knew which model they received, whether graders were blind to treatment condition, or whether the analyst was blinded during analysis."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 300 participants and 1,800 tasks, but regression tables show varying N (1,500, 1,392, 1,391) with no explanation of attrition or why observations were excluded."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 footnote states: 'the inference costs for all 300 translators completing nearly 1,400 tasks with AI assistance combined came to significantly less than a dollar.' Also mentions $1 per 100,000 syllables for leading models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget for the experiment is stated. While inference costs are mentioned, the total experimental cost (participant payments, grader payments, infrastructure) is not reported."
       }
     }
   },
   "claims": [
     {
       "claim": "A 10x increase in model training compute improves task completion speed by 12.3%",
       "evidence": "Section 3.2, Table 3: regression coefficient of -51.35 seconds per log10 compute unit (p=0.001), corresponding to 12.3% reduction in time.",
       "supported": "strong"
     },
     {
       "claim": "A 10x increase in model compute improves grades by 0.18 standard deviations (0.25 points on 7-point scale)",
       "evidence": "Section 3.3, Table 4: regression coefficient of 0.2584 grade points per log10 compute (p=0.000).",
       "supported": "strong"
     },
     {
       "claim": "A 10x increase in model compute increases earnings per minute by 16.1%",
       "evidence": "Section 3.4, Table 5: regression coefficient of 0.1921 earnings per minute per log10 compute (p=0.001).",
       "supported": "strong"
     },
     {
       "claim": "Productivity gains are 4x larger for lower-skilled workers (21.1% vs 4.9% time reduction per 10x compute)",
       "evidence": "Section 3.5, Table 1: interaction term Skill×logmodelcompute = -69.399 (p=0.017). High-skill 4.9% reduction, low-skill 21.1% reduction.",
       "supported": "strong"
     },
     {
       "claim": "Continued model scaling could boost U.S. productivity by at least 6.9% over the next decade",
       "evidence": "Section 4: calculation using 19.9% task exposure × 61.2% productivity effect × 57% labor share = 6.95%. Based on Hulten's theorem framework from Acemoglu (2024).",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["rct"],
   "key_findings": "In a pre-registered RCT with 300 professional translators and 13 LLMs, a 10x increase in model training compute reduced task completion time by 12.3%, improved quality grades by 0.18 SD, and increased earnings per minute by 16.1%. Gains were 4x larger for lower-skilled workers. The paper extrapolates from these translation-specific findings to estimate 6.9% U.S. productivity growth over the next decade, an order of magnitude higher than Acemoglu (2024).",
   "red_flags": [
     {
       "flag": "Extrapolation far beyond tested domain",
       "detail": "The 6.9% aggregate U.S. productivity estimate in Section 4 extrapolates from a single task domain (short translation tasks) to the entire economy. The paper acknowledges translation may be the most AI-exposed occupation, yet uses the same scaling ratio for all tasks. The claim of 'at least 6.9%' (framed as a lower bound) rests on strong assumptions about scaling law generalizability."
     },
     {
       "flag": "Unexplained observation attrition",
       "detail": "300 participants × 6 tasks = 1,800 potential observations, but Table 2 shows 1,500 (any-AI analysis) and Tables 3-5 show 1,392/1,391 (scaling analysis). The difference between 1,800 and 1,500 and the further reduction to 1,392 is never explained. Missing data could bias results."
     },
     {
       "flag": "Short artificial tasks may not represent professional work",
       "detail": "Tasks averaged ~10 minutes each. Participants rated similarity to professional work at only 3.53/5. Real translation involves longer documents, domain expertise, client interactions, and iterative revision — the experimental setup may overstate productivity gains for simple tasks."
     },
     {
       "flag": "Funder alignment with findings",
       "detail": "Open Philanthropy, which funded the research, is a major funder of AI-related research and has organizational positions on AI's transformative potential. The paper's core finding (large productivity gains from AI scaling) aligns with the funder's worldview."
     },
     {
       "flag": "Recruitment via gig platforms",
       "detail": "Participants were recruited from Freelancer and Fiverr. Gig-platform translators may differ systematically from translators working in agencies or in-house roles in terms of skill level, AI familiarity, and task approach. The paper does not discuss this selection bias."
     }
   ],
   "cited_papers": [
     {
       "title": "The Simple Macroeconomics of AI",
       "authors": ["Daron Acemoglu"],
       "year": 2024,
       "relevance": "Key framework paper for estimating aggregate productivity gains from AI; this paper directly extends and critiques its estimates."
     },
     {
       "title": "Experimental Evidence on the Productivity Effects of Generative Artificial Intelligence",
       "authors": ["Shakked Noy", "Whitney Zhang"],
       "year": 2023,
       "doi": "10.1126/science.adh2586",
       "relevance": "RCT finding 37% productivity improvements from ChatGPT for professional writing tasks; used as a key parameter in the aggregate estimate."
     },
     {
       "title": "Generative AI at Work",
       "authors": ["Erik Brynjolfsson", "Danielle Li", "Lindsey R. Raymond"],
       "year": 2023,
       "relevance": "Real-world case study of AI productivity effects in call centers (14% improvement); used as a key parameter in the aggregate estimate."
     },
     {
       "title": "Navigating the Jagged Technological Frontier: Field Experimental Evidence of the Effects of AI on Knowledge Worker Productivity and Quality",
       "authors": ["Fabrizio Dell'Acqua"],
       "year": 2023,
       "relevance": "BCG consultant field experiment showing heterogeneous productivity effects of GPT-4 across task types."
     },
     {
       "title": "Scaling Laws for Neural Language Models",
       "authors": ["Jared Kaplan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws paper showing consistent relationship between compute and cross-entropy loss; this paper extends scaling laws to economic outcomes."
     },
     {
       "title": "GPTs are GPTs: An Early Look at the Labor Market Impact Potential of Large Language Models",
       "authors": ["Tyna Eloundou"],
       "year": 2023,
       "arxiv_id": "2303.10130",
       "relevance": "Estimates 19.9% of tasks exposed to AI, directly used as a parameter in the aggregate productivity calculation."
     },
     {
       "title": "Evidence of a Log Scaling Law for Political Persuasion with Large Language Models",
       "authors": ["Kobi Hackenburg"],
       "year": 2024,
       "arxiv_id": "2406.14508",
       "relevance": "Derives scaling laws for LLM persuasion performance, the most closely related prior work on economic/social scaling laws."
     },
     {
       "title": "Beyond AI Exposure: Which Tasks are Cost-Effective to Automate with Computer Vision?",
       "authors": ["Maja Svanberg", "Wensu Li", "Martin Fleming", "Brian Goehring", "Neil Thompson"],
       "year": 2024,
       "relevance": "Estimates 23% of automatable tasks are economically feasible; this paper argues the estimate is too conservative for LLM tasks."
     },
     {
       "title": "Productivity Assessment of Neural Code Completion",
       "authors": ["Albert Ziegler"],
       "year": 2022,
       "relevance": "GitHub Copilot productivity assessment showing double-digit code completion improvements; one of the key AI productivity studies referenced."
     }
   ]
 }