ai-research-survey

scan.json (22753B)

---

 {
   "scan_version": 2,
   "active_modules": [],
   "paper": {
     "title": "GPTs are GPTs: An Early Look at the Labor Market Impact Potential of Large Language Models",
     "authors": ["Tyna Eloundou", "Sam Manning", "Pamela Mishkin", "Daniel Rock"],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2303.10130"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or URL is provided in the paper. The exposure rubric is described but no analysis code is released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses public O*NET and BLS data but does not release its own annotated exposure dataset. The human and GPT-4 annotations that form the core contribution are not made publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependencies, or software versions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The rubric is given in Appendix A.1 but there are no instructions for replicating the annotation process or analysis pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Table 3, Figures 3-5) report means and standard deviations of exposure scores but no confidence intervals or error bars on the key exposure estimates."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Table 9 reports OLS regressions with standard errors and significance levels (*, **, ***) for validation against prior measures. Table 5 similarly reports significance for skill regressions."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes in context throughout — e.g., '15% of all worker tasks could be completed significantly faster,' '80% of workers have at least 10% of tasks affected,' with baseline percentages across exposure levels (α, β, ζ) in Tables 3, 6, 7, 10."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 1,016 occupations from O*NET and 19,265 tasks but does not justify why these sample sizes are adequate for the claims made. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports standard deviations for all exposure measures. Table 4 identifies occupations with highest variance. Table 8 provides std dev for all prior measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 and Table 9 compare the new LLM exposure measure against multiple prior efforts: Frey & Osborne, Webb, Brynjolfsson et al. (SML), Felten et al., and Acemoglu & Autor routine task measures."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Felten et al. (2023) and Brynjolfsson et al. (2023), which are contemporary. Older measures like Frey & Osborne (2017) are included for historical comparison with justification."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper effectively ablates by comparing three exposure levels (α = LLM only, β = weighted, ζ = LLM + software) and comparing human vs. GPT-4 annotations, and two different GPT-4 rubric prompts (Table 2)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses three exposure measures (α, β, ζ), agreement rates, and Pearson correlations (Table 2). Regressions use R² and coefficient significance."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human annotators labeled DWAs and tasks using the exposure rubric (Section 3.3). The authors and experienced annotators from OpenAI's alignment work provided human ratings."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a predictive modeling paper — it produces descriptive exposure measures, not predictions to be tested on held-out data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by occupation (Table 4), Job Zone (Table 6, Figure 5), education level (Table 10), on-the-job training (Table 7), skill importance (Table 5), and industry (Figures 6-7)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4.3 discusses sources of disagreement, including tasks where annotators got 'stuck.' Table 11 lists 34 occupations with zero exposure. The paper acknowledges discrepancies between human and GPT-4 labels."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that AI Occupational Exposure Score from Felten et al. is 'not correlated' with their measure in most specifications (Table 9). Also reports that LLM exposure appears uncorrelated with recent productivity growth."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about 80% of workforce having 10% of tasks affected, 19% having 50% affected, ~15% direct exposure, 47-56% with software — all supported by Tables 3 and summary statistics in Section 4.1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to frame findings as 'exposure' and 'potential impact' rather than causal effects. Section 3.3 explicitly states exposure is 'a proxy for potential economic impact without distinguishing between labor-augmenting or labor-displacing effects.' The paper avoids causal language."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 explicitly limits to the US, acknowledges the forward-looking nature of predictions, and notes that 'our focus on the United States restricts the generalizability of our findings to other nations.' The abstract says 'U.S. labor market' not labor markets generally."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 discusses multiple limitations including annotator subjectivity (3.4.1), sensitivity to rubric wording (3.4.2), validity of the task-based framework, and sources of disagreement (3.4.3). Section 6.1 discusses adoption bottlenecks and confounds."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 explicitly states 'We define exposure as a proxy for potential economic impact without distinguishing between labor-augmenting or labor-displacing effects.' The paper repeatedly distinguishes technical exposure from actual adoption, productivity, or automation outcomes."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'an early version of GPT-4' was used for classification (Section 3.3) without specifying the exact version, snapshot date, or API endpoint. Section 1 notes 'GPT-3.5 family but not in the GPT-4 family' for the rubric definition but not the classifier version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full exposure rubric/prompt used for annotation is provided in Appendix A.1, including detailed examples and category definitions. This is the prompt used for GPT-4 classification (with acknowledged slight modifications, Section 3.4.2)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for the GPT-4 classification runs."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. GPT-4 is used as a direct classifier via prompting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the O*NET data (19,265 tasks, 2,087 DWAs). Section 3.2 describes BLS data linkage via crosswalk. Section 3.3 describes the aggregation from DWA to task to occupation level with core/supplemental weighting."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 'Limitations of our methodology' contains four substantive subsections: subjective judgments (3.4.1), measuring with GPT-4 (3.4.2), additional weaknesses (3.4.3). Section 6.3 adds further limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4.1 discusses annotator homogeneity ('not occupationally diverse'). Section 3.4.2 discusses prompt sensitivity. Section 3.4.3 identifies specific issues: validity of task-based framework, annotator lack of occupation expertise, and forward-looking uncertainty."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 states 'our focus on the United States restricts the generalizability.' The paper explicitly says it does not predict adoption timelines (abstract) and 'does not consider total factor productivity or capital input potential' (Section 4). It notes exposure ≠ automation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The annotated exposure labels are not released. O*NET and BLS data are public but the paper's core contribution — the human and GPT-4 annotations — is not available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1-3.3 describes data sources (O*NET 27.2, BLS 2020-2021), annotation procedure (authors + contracted annotators), and the exposure rubric. The crosswalk method is specified."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes annotators: 'The authors personally labeled a large sample' and 'enlisted experienced human annotators who have reviewed GPT-3, GPT-3.5 and GPT-4 outputs as part of OpenAI's alignment work.' Acknowledgments name specific annotators."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from O*NET DWAs → task aggregation → occupation aggregation with core/supplemental weighting is described. Three measures (α, β, ζ) are defined. Human vs. GPT-4 annotation tracks are described. BLS crosswalk is specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. Acknowledgments thank individuals for feedback but do not mention grants or funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are prominently listed: three authors from OpenAI, one from OpenResearch, one from University of Pennsylvania. The OpenAI affiliation is clear."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three of four authors are OpenAI employees. OpenAI has a direct financial interest in demonstrating the broad economic impact of LLMs/GPTs. The funder (effectively OpenAI as employer) is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. OpenAI employees evaluating the economic impact potential of OpenAI's products represents a clear potential conflict that is not explicitly declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. GPT-4 is used as a classifier/annotator, not tested on its knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of model capability is performed. GPT-4 is used as a labeling tool, not evaluated on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The paper uses GPT-4 as an annotator, not as a test subject."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the experimental sense. Human annotators labeled tasks but were not study participants. This is a labor market analysis, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Annotators were paid workers performing a labeling task, not research subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Annotator characteristics are briefly described (familiar with LLM capabilities, OpenAI alignment annotators) but this is not a human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper used GPT-4 to classify 19,265 tasks but does not report the API cost or compute time required."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated for the GPT-4 annotation runs or the statistical analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "About 80% of the U.S. workforce could have at least 10% of their work tasks affected by LLMs",
       "evidence": "Table 3 and Section 4.1: based on β exposure values aggregated to occupation level with core/supplemental task weighting.",
       "supported": "moderate"
     },
     {
       "claim": "Approximately 19% of workers may see at least 50% of their tasks impacted by LLMs and LLM-powered software",
       "evidence": "Section 4.1: '19% of workers are in an occupation where over half of its tasks are labeled as exposed' based on β values.",
       "supported": "moderate"
     },
     {
       "claim": "About 15% of all worker tasks could be completed significantly faster with LLM alone",
       "evidence": "Table 3: mean α values of 0.14-0.15 from both human and GPT-4 annotations.",
       "supported": "moderate"
     },
     {
       "claim": "With LLM-powered software, 47-56% of all tasks could be completed significantly faster",
       "evidence": "Table 3: mean ζ values of 0.47 (human) and 0.56 (GPT-4) at occupation level.",
       "supported": "weak"
     },
     {
       "claim": "Higher-income jobs face greater exposure to LLM capabilities",
       "evidence": "Figure 4 binscatter, Table 6 Job Zone analysis, Table 10 education-based analysis all show increasing exposure with wage/education level.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs exhibit traits of general-purpose technologies",
       "evidence": "Section 6.1 argues LLMs meet three GPT criteria: improvement over time (literature review), pervasiveness (exposure analysis), complementary innovations (α vs ζ gap). Evidence is conceptual rather than rigorous testing of GPT criteria.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["observational", "qualitative"],
   "key_findings": "Using human and GPT-4 annotations on O*NET occupation data, the paper estimates ~15% of US worker tasks are directly exposed to LLMs, rising to 47-56% with LLM-powered software. Higher-wage occupations show greater exposure, reversing patterns seen with prior automation technologies. The exposure measure correlates positively with prior software/AI exposure scores but negatively with manual/robotics measures, with 28-40% unexplained variance. The authors argue LLMs qualify as general-purpose technologies based on their pervasiveness and complementary innovation potential.",
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "Three of four authors are OpenAI employees. The paper argues for the broad economic impact of GPTs/LLMs — OpenAI's core product. While the paper frames results as 'exposure' rather than benefits, the framing still serves OpenAI's narrative that GPTs are transformative general-purpose technologies."
     },
     {
       "flag": "Annotator pool not occupationally diverse",
       "detail": "The paper acknowledges in Section 3.4.1 that annotators 'are not occupationally diverse, potentially leading to biased judgments regarding LLMs' reliability and effectiveness in performing tasks within unfamiliar occupations.' Annotators were OpenAI alignment workers, likely biased toward overestimating LLM capabilities."
     },
     {
       "flag": "Forward-looking exposure estimates presented as findings",
       "detail": "The ζ measure (LLM + software) requires imagining future software that doesn't yet exist. This makes 47-56% exposure figure speculative by design, yet it is the headline number in the abstract. The β measure weights this at 50%."
     },
     {
       "flag": "No release of annotation data",
       "detail": "The core contribution — human and GPT-4 exposure labels for 19,265 tasks — is not released, preventing independent verification of the coding decisions that drive all results."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Chen, M.", "Tworek, J.", "Jun, H."],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational code generation benchmark (Codex/HumanEval) widely used in LLM programming evaluations."
     },
     {
       "title": "The impact of ai on developer productivity: Evidence from github copilot",
       "authors": ["Peng, S.", "Kalliamvakou, E.", "Cihon, P.", "Demirer, M."],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "RCT on GitHub Copilot productivity effects — directly relevant to LLM programming productivity claims."
     },
     {
       "title": "Experimental evidence on the productivity effects of generative artificial intelligence",
       "authors": ["Noy, S.", "Zhang, W."],
       "year": 2023,
       "relevance": "Experimental evidence on generative AI productivity effects on writing tasks."
     },
     {
       "title": "On the opportunities and risks of foundation models",
       "authors": ["Bommasani, R.", "Hudson, D. A.", "Adeli, E."],
       "year": 2021,
       "arxiv_id": "2108.07258",
       "relevance": "Comprehensive survey of foundation model capabilities, risks, and societal implications."
     },
     {
       "title": "Augmented language models: a survey",
       "authors": ["Mialon, G.", "Dessì, R.", "Lomeli, M."],
       "year": 2023,
       "arxiv_id": "2302.07842",
       "relevance": "Survey of augmented LLMs including tool use and retrieval — relevant to LLM-powered software capabilities."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Schick, T.", "Dwivedi-Yu, J.", "Dessì, R."],
       "year": 2023,
       "arxiv_id": "2302.04761",
       "relevance": "Key paper on LLM tool use capabilities that enable the LLM-powered software discussed in this paper."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Ouyang, L.", "Wu, J.", "Jiang, X."],
       "year": 2022,
       "arxiv_id": "2203.02155",
       "relevance": "RLHF paper fundamental to the instruction-following capabilities that make LLMs useful for task completion."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Brown, T.", "Mann, B.", "Ryder, N."],
       "year": 2020,
       "relevance": "GPT-3 paper establishing large-scale language model capabilities for diverse tasks."
     },
     {
       "title": "The future of employment: How susceptible are jobs to computerisation?",
       "authors": ["Frey, C. B.", "Osborne, M. A."],
       "year": 2017,
       "relevance": "Seminal automation exposure study that this paper directly builds upon and compares against."
     },
     {
       "title": "What can machines learn, and what does it mean for occupations and the economy?",
       "authors": ["Brynjolfsson, E.", "Mitchell, T.", "Rock, D."],
       "year": 2018,
       "relevance": "Suitability for Machine Learning (SML) rubric that this paper's methodology directly extends to LLMs."
     }
   ]
 }