ai-research-survey

scan.json (21021B)

---

 {
   "paper": {
     "title": "GATE: An Integrated Assessment Model for AI Automation",
     "authors": ["Ege Erdil", "Andrei V. Potlogea", "Tamay Besiroglu", "Edu Roldan", "Anson Ho"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.04941"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["theoretical"],
   "key_findings": "GATE is an integrated assessment model combining a compute-based AI development module, an AI automation module mapping effective compute to task automation, and a semi-endogenous growth macroeconomic module. The model features endogenous investment, adjustment costs, R&D externalities, and uncertainty add-ons. It is implemented as an interactive sandbox allowing users to simulate economic effects of AI automation under different parameter assumptions. The paper's primary contribution is bridging AI capability modeling with macroeconomic growth theory in a unified dynamic framework.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides an interactive sandbox at www.epoch.ai/GATE (mentioned in abstract, Section 1, Section 2, and Section 7). While not traditional source code, the model is implemented and publicly accessible."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides extensive parameter tables with ranges, defaults, and empirical justifications. The model sandbox allows users to explore all parameter settings. All calibration data is documented in the appendices."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or technical requirements for running the model are provided. The sandbox is web-based but no technical details about its implementation are given."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the mathematical model is fully specified, there are no step-by-step instructions for reproducing the numerical solver. Appendix C describes the solution approach (gradient descent) but lacks implementation details sufficient for reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical modeling paper presenting a simulation framework, not an empirical study with statistical results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical modeling paper; no comparative empirical claims requiring significance tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical modeling paper; no empirical effect sizes to report."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical paper with no samples."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical modeling paper; no experimental runs with variance to report."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not compare GATE against other integrated assessment models or prior economic models of AI automation (e.g., Davidson 2023, Korinek and Suh 2024) in terms of outputs or predictions. It only discusses them conceptually."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No quantitative comparison against contemporary models is provided."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "While the model has optional add-ons (R&D externalities, uncertainty), no systematic ablation is presented showing how each module affects outcomes. The paper describes functionality but does not present ablation results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 describes multiple output variables: GWP growth, consumption, capital investment, automation fraction, training run size, compute allocation, hardware/software efficiency trajectories (Figure 5 shows 9 different output panels)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for a theoretical economic model."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical modeling paper; no test sets involved."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 provides breakdowns across multiple output categories. The model tracks per-module outputs (AI development, automation, macro) separately."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Limitations) extensively discusses failure modes and limitations: lack of non-AI TFP growth, stylized effective compute abstraction, failure to incorporate data production, stylized labor reallocation, omission of market structure, static task space, and other bottlenecks."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report scenarios where the model fails to converge, produces unrealistic outputs, or scenarios where the framework breaks down. Appendix D mentions convergence issues but does not present them as results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims GATE combines three ingredients (compute-based AI development, automation framework, semi-endogenous growth model) in an interactive sandbox. The paper fully describes all three modules and references the sandbox URL."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper presents a theoretical model framework. It does not make empirical causal claims; it describes structural relationships within the model."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 extensively bounds the model's scope, noting it omits non-AI TFP growth, data constraints, realistic labor market frictions, market structure, task heterogeneity, and other bottlenecks. The paper is clear about what GATE does and does not model."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses alternative modeling approaches and mechanisms that could yield different outcomes: endogenous TFP, data constraints, market structures, directed technical change, and additional bottlenecks. Section 9 outlines four key directions for future work addressing these alternatives."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical paper with no empirical measurements to distinguish from proxy outcomes."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use any AI models for evaluation. It is a theoretical economic model."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in this paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides extensive parameter tables (Tables 3-10) with ranges, default values, units, and empirical justifications for all model parameters across general economics, hardware R&D, software R&D, compute investment, compute stock, runtime compute, and automation categories."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents how each parameter was derived from empirical sources (e.g., GPU pricing, World Bank data, meta-analyses) with specific citations and calculations in Appendix D."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 ('Limitations') is a dedicated 5-page section discussing structural simplifications and parametric uncertainty."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 provides highly specific threats: the effective compute abstraction neglects serial vs parallel constraints (citing Erdil and Schneider-Joseph 2024), data scarcity as a binding constraint (citing Villalobos et al. 2022), labor reallocation extremes miss retraining dynamics, and social planner framework mischaracterizes competitive investment dynamics."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states what the model does NOT capture: non-AI TFP growth, data production, intermediate labor reallocation, market structures, task heterogeneity (cognitive vs manual), new task creation, and various bottlenecks. This is thorough boundary-setting."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While parameter calibrations reference external data sources (World Bank, NVIDIA specs, meta-analyses), the raw data used for parameter estimation is not provided or linked as a dataset."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D describes in detail how each parameter was derived, citing specific data sources (World Bank GDP, labor force stats, NVIDIA H100 specs, meta-analyses of risk aversion, depreciation rates, etc.)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data comes from public economic statistics and hardware specifications."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D documents the derivation pipeline for each parameter, showing intermediate calculations (e.g., hardware efficiency: H100 price → FLOP/s → FLOP/year/$ calculation)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed. The paper is from Epoch AI but no grants or funding sources are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states it is from Epoch AI, with detailed author contributions in Section A (Appendix)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Epoch AI is an AI forecasting organization with a stake in AI development timelines being taken seriously. No explicit funding disclosure makes independence impossible to assess."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on any benchmark. It is a theoretical economic model."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model evaluation on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model evaluation on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical paper; no inference costs to report."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical paper; no computational experiments with budgets to report."
       }
     }
   },
   "claims": [
     {
       "claim": "GATE is the first integrated assessment model combining compute-based AI development, AI automation framework, and semi-endogenous growth model.",
       "evidence": "Section 1 and Section 2 describe the three-module architecture. The paper claims novelty in bringing these together: 'GATE combines three key ingredients that have not been brought together in previous work' (Abstract).",
       "supported": "moderate"
     },
     {
       "claim": "The AI development module is the most novel and significant contribution of GATE.",
       "evidence": "Sections 3.1-3.7 provide detailed modeling of effective compute accumulation, training-inference tradeoffs, and R&D dynamics. The paper states this explicitly in Sections 2 and 3.",
       "supported": "moderate"
     },
     {
       "claim": "Effective compute is a sufficient statistic for tracking AI capabilities.",
       "evidence": "Section 3.1 and Appendix B cite scaling laws (Kaplan et al. 2020, Hoffmann et al. 2022), algorithmic progress estimates (Hernandez and Brown 2020, Ho et al. 2024), and Sutton's bitter lesson. However, Section 8 acknowledges this is a significant simplification.",
       "supported": "moderate"
     },
     {
       "claim": "Hardware and software R&D display a key asymmetry: software improvements enhance all compute while hardware improvements only affect new compute.",
       "evidence": "Section 3.4 and equation (12) in Section 3.7 formalize this asymmetry in the law of motion for effective compute.",
       "supported": "strong"
     },
     {
       "claim": "The model can be solved numerically via gradient descent on the social planner's value function.",
       "evidence": "Appendix C describes the solution approach with discretization (20 task grid points, 1-year time steps, 80-year planning horizon) and gradient descent over 5 degrees of freedom per timestep.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No empirical validation",
       "detail": "The paper presents a theoretical model without any empirical validation against historical data. No backtesting against observed compute investment, GDP growth, or automation levels is shown. Figure 5 shows illustrative outputs but no comparison to actual economic trajectories."
     },
     {
       "flag": "Incomplete solver documentation",
       "detail": "Appendix C contains placeholder comments in brackets ('[If there is anything we do explicitly to avoid the problem of getting stuck at a local optimum...]', '[In a conversation with Ege he mentioned...]'), suggesting the paper was published with incomplete documentation of the solution method."
     },
     {
       "flag": "Self-evaluating organization",
       "detail": "Epoch AI is an AI forecasting organization presenting its own model. The model's default parameters and framing choices (e.g., full automation as a central scenario) align with the organization's public positioning on AI timelines."
     },
     {
       "flag": "Single effective compute dimension is acknowledged as inadequate",
       "detail": "Section 8 acknowledges that reducing algorithmic progress to a single dimension fails to capture serial vs parallel constraints, scale-dependent improvements, and the distinction between cost-lowering and capability-expanding innovations. This is a fundamental limitation of the core modeling approach."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan", "Tom B Brown"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws work showing AI performance predictable from compute, central to GATE's compute-based approach."
     },
     {
       "title": "An empirical analysis of compute-optimal large language model training",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch"],
       "year": 2022,
       "relevance": "Chinchilla scaling laws establishing compute-optimal training, used to justify GATE's effective compute framework."
     },
     {
       "title": "Algorithmic progress in language models",
       "authors": ["Anson Ho", "Tamay Besiroglu", "Ege Erdil"],
       "year": 2024,
       "arxiv_id": "2403.05812",
       "relevance": "Estimates algorithmic efficiency doubling times for LLMs, directly calibrates GATE's software efficiency parameters."
     },
     {
       "title": "Scenarios for the transition to AGI",
       "authors": ["Anton Korinek", "Donghyun Suh"],
       "year": 2024,
       "relevance": "Similar compute-to-automation framework that GATE builds upon, key related work for economic modeling of AI automation."
     },
     {
       "title": "What a compute-centric framework says about AI takeoff speeds",
       "authors": ["Tom Davidson"],
       "year": 2023,
       "relevance": "Open Philanthropy report introducing the FLOP gap concept and compute-centric AI forecasting, directly inspires GATE's automation module."
     },
     {
       "title": "Explosive growth from AI automation: A review of the arguments",
       "authors": ["Tamay Besiroglu", "Ege Erdil"],
       "year": 2023,
       "arxiv_id": "2309.11690",
       "relevance": "Reviews arguments for AI-driven explosive economic growth, provides theoretical foundations for GATE's growth dynamics."
     },
     {
       "title": "Data movement limits to frontier model training",
       "authors": ["Ege Erdil", "David Schneider-Joseph"],
       "year": 2024,
       "arxiv_id": "2411.01137",
       "relevance": "Shows fundamental data-movement constraints limit training beyond 1e28-1e31 FLOP, cited as limitation of GATE's single effective compute metric."
     },
     {
       "title": "The rising costs of training frontier AI models",
       "authors": ["Ben Cottier", "Robi Rahman", "Loredana Fattorini"],
       "year": 2024,
       "arxiv_id": "2405.21015",
       "relevance": "Empirical analysis of AI training costs used to calibrate GATE's compute investment parameters."
     },
     {
       "title": "GPTs are GPTs: An early look at the labor market impact potential of large language models",
       "authors": ["Tyna Eloundou", "Sam Manning", "Pamela Mishkin", "Daniel Rock"],
       "year": 2023,
       "arxiv_id": "2303.10130",
       "relevance": "Assesses LLM exposure across occupations, relevant to GATE's task automation framework."
     },
     {
       "title": "Trading off compute in training and inference",
       "authors": ["Pablo Villalobos", "David Atkinson"],
       "year": 2023,
       "relevance": "Empirical estimates of training-inference compute tradeoff used to calibrate GATE's inference multiplier parameter."
     }
   ]
 }