ai-research-survey

scan.json (21441B)

---

 {
   "paper": {
     "title": "Artificial or Human Intelligence?",
     "authors": ["Eric Gao"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.02879"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": false,
         "answer": false,
         "justification": "This is a purely theoretical economics paper with mathematical proofs and no computational experiments. There is no code to release."
       },
       "data_released": {
         "applies": false,
         "answer": false,
         "justification": "No empirical data is collected or used. The paper is purely theoretical with mathematical models."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments are performed. The paper contains only mathematical derivations and proofs."
       },
       "reproduction_instructions": {
         "applies": false,
         "answer": false,
         "justification": "As a purely theoretical paper, reproduction consists of verifying mathematical proofs, which are provided in the text itself. No experimental reproduction instructions are needed."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "No empirical results are reported. The paper contains only theoretical derivations."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No empirical comparisons are made. All results are mathematical propositions."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No empirical effect sizes. Results are theoretical comparative statics expressed as inequalities on partial derivatives."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Theoretical paper with no samples."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No empirical runs or experiments. All results are analytical."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical model, not an evaluation of a system. The paper compares its model's predictions to related theoretical and empirical work in the related literature section, but does not perform a baseline comparison in the experimental sense."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No experimental baselines. The paper is purely theoretical."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate. The paper derives comparative statics for different model parameters, which serves a similar conceptual role but is not an ablation study."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No evaluation metrics are used. Results are mathematical propositions about equilibrium behavior."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate. The paper is purely theoretical."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No data or test sets used."
       },
       "per_category_breakdown": {
         "applies": false,
         "answer": false,
         "justification": "No empirical results to break down by category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Discussion) discusses extensively the limitations and cases where the model's predictions may not hold, including multi-dimensional skills, heterogeneous student circumstances, and the simplistic payoff function. The paper notes 'the model considered in this paper is extremely simplistic' and lists several ways the model could fail to capture real-world dynamics."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that advances in AI technology do not always lead to increases in human learning (Section 5, 7), and that AI can increase inequality between students. These are negative findings about AI in education. Section 5.1 explicitly shows conditions under which AI advances make things worse for students who use AI as a solver."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported by the theoretical results. Key claims: (1) AI alters student incentives to learn — supported by Proposition 1 and comparative statics in Section 5; (2) discontinuous gap between students above/below AI frontier — supported by Proposition 1's proof of discontinuity at threshold T; (3) increasing AI-free assignments can counteract misspecification — supported by Section 6's analysis showing λ = p/p' restores efficiency."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims within a formal model (e.g., 'increasing the accuracy of AI decreases the marginal benefit to learning for students who use AI as a solver'). These are derived as mathematical consequences of the model assumptions through comparative statics (Section 5), which is the appropriate methodology for theoretical economics. The causal claims are valid within the model's assumptions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Discussion) extensively bounds the model's generalizability: 'the model considered in this paper is extremely simplistic. There is only one dimension of problems but students often require multiple complementary skills.' The paper lists specific dimensions not covered: multi-dimensional skills, heterogeneous time constraints, different degrees of misspecification across students, and differences between classroom and real-world problem distributions."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses multiple alternative mechanisms that could explain observed patterns differently: complementarity vs substitutability of skills, nuanced AI usage patterns (helpers sometimes using AI as solver for routine tasks), heterogeneous student time constraints, and self-selection into courses as a confound (footnote 3). Section 3.2 also discusses the alternative view that emergence is an artifact of discontinuous benchmarks rather than true model behavior (Schaeffer et al., 2023)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No LLM or AI model is used in the experiments. The paper theoretically models AI capabilities with abstract parameters d and p."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. This is a theoretical paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments requiring hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding. This is a theoretical paper."
       },
       "data_preprocessing_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data preprocessing. The paper is purely theoretical with no data."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Discussion) functions as a substantial limitations section, discussing at length the simplifications of the model and open questions. It spans roughly two pages and identifies multiple specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 identifies specific threats: the one-dimensional problem space ('students often require multiple complementary skills'), the simplistic payoff function ('students may make more nuanced decisions about when and how to use AI tools'), the single misspecification parameter ('different students may be misspecified to different degrees'), and the gap between classroom and real-world problem distributions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states what the model does NOT address: multi-dimensional skills, heterogeneous time availability, varying degrees of misspecification, the difference between classroom and real-world problem distributions, and the feedback loop where AI learns from humans. The paper says 'Most interesting would be to take the model to data' — explicitly acknowledging the paper provides no empirical validation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": false,
         "answer": false,
         "justification": "No data is collected. The paper is purely theoretical."
       },
       "data_collection_described": {
         "applies": false,
         "answer": false,
         "justification": "No data collection. Purely theoretical paper."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No participants or data collection. Purely theoretical."
       },
       "data_pipeline_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data pipeline. Purely theoretical paper."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure is present. The paper acknowledges individuals for 'helpful comments and discussions' but does not mention any funding source or grant support."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation is disclosed: 'Department of Economics, MIT' with email ericgao@mit.edu."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, and this appears to be unfunded academic work by a single MIT researcher."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on any benchmark. The paper is a theoretical model."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation on benchmarks. Purely theoretical."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmarks used. Purely theoretical paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Purely theoretical paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper. No method with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Purely theoretical paper. No computation performed."
       }
     }
   },
   "claims": [
     {
       "claim": "Access to AI creates a discontinuous gap in student ability at a threshold type, where students below the threshold use AI as a solver (lower ability than AI) and those above use it as a helper (higher ability than AI).",
       "evidence": "Proposition 1 (Section 4) proves the existence of a unique threshold type T with a discontinuity in the ability mapping A(t), showing lim(t→T-) A(t) = as(T) < d < ah(T) = lim(t→T+) A(t).",
       "supported": "strong"
     },
     {
       "claim": "Increasing AI accuracy (reducing hallucinations) decreases the marginal benefit of learning for students who use AI as a solver but does not impact those who use AI as a helper.",
       "evidence": "Section 5.1 shows that the marginal benefit for solvers is (1-p) which decreases as p increases, while the marginal benefit for helpers is 1 regardless of p.",
       "supported": "strong"
     },
     {
       "claim": "The more advanced AI becomes (higher d and p), the more difficult it is for further AI advances to increase human learning.",
       "evidence": "Section 5.2 derives conditions for when the solver-helper threshold T increases with d and p, showing the required complementarities between ability and AI parameters must be larger when d and p are already high.",
       "supported": "strong"
     },
     {
       "claim": "Instructors can counteract student misspecification about AI accuracy by placing weight on assignments that do not permit AI usage, with optimal weight λ = p/p'.",
       "evidence": "Section 6 derives that when students overestimate AI accuracy (p' > p), setting λ = p/p' in the assignment weight makes the misspecified student's problem coincide with the true problem, restoring efficiency.",
       "supported": "strong"
     },
     {
       "claim": "AI advances can increase inequality between students of different types, as it is more difficult for advances to incentivize humans who use AI as a solver to invest in higher ability compared to helpers.",
       "evidence": "Section 5.1 shows the condition for increasing differences in ability with p is harder to satisfy for solvers (requiring ∂²c/∂a∂p < -1) than for helpers (requiring ∂²c/∂a∂p < 0).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["theoretical"],
   "key_findings": "This theoretical economics paper models how AI tools with hallucinations and emergent capabilities affect student incentives to invest in learning. The key finding is that AI access creates a discontinuous gap in student ability: students below a threshold type use AI as a solver (with lower ability than AI) while those above the threshold use it as a helper (with higher ability). Advances in AI accuracy can increase this inequality by reducing learning incentives for solver-students more than helper-students. The paper shows that educators can counteract student overestimation of AI accuracy by optimally weighting assignments that prohibit AI usage.",
   "red_flags": [
     {
       "flag": "No empirical validation",
       "detail": "The paper is entirely theoretical with no empirical tests of its predictions. The author acknowledges this ('Most interesting would be to take the model to data') but the model's predictions remain untested. The stylized assumptions (uniform problem distribution, sharp AI cutoff, one-dimensional ability) may not hold in practice."
     },
     {
       "flag": "Strong simplifying assumptions",
       "detail": "The model assumes one-dimensional problems, one-dimensional ability, uniform problem distribution, perfect human accuracy within their ability range, and a sharp AI cutoff. These assumptions drive the key results (especially the discontinuity at threshold T) but are acknowledged as unrealistic by the author in Section 7."
     }
   ],
   "cited_papers": [
     {
       "title": "Generative AI Can Harm Learning",
       "authors": ["Hamsa Bastani", "Osbert Bastani", "Alp Sungu", "Haosen Ge", "Özge Kabakcı", "Rei Mariman"],
       "year": 2024,
       "doi": "10.2139/ssrn.4895486",
       "relevance": "Field experiment on GPT-4 in high-school math classes showing 17% decrease in exam performance when AI is later removed, directly relevant to AI impact on learning."
     },
     {
       "title": "Generative AI at Work",
       "authors": ["Erik Brynjolfsson", "Danielle Li", "Lindsey Raymond"],
       "year": 2025,
       "doi": "10.1093/qje/qjae044",
       "relevance": "Empirical study of AI adoption among customer-support agents showing AI benefits lower-ability workers more, foundational evidence for AI productivity effects."
     },
     {
       "title": "Canaries in the Coal Mine? Six Facts about the Recent Employment Effects of Artificial Intelligence",
       "authors": ["Erik Brynjolfsson", "Bharat Chandar", "Ruyu Chen"],
       "year": 2025,
       "relevance": "Uses payroll data to study AI impact on labor markets, finding young workers in AI-exposed fields harmed most, relevant to AI workforce effects."
     },
     {
       "title": "Experimental evidence on the productivity effects of generative artificial intelligence",
       "authors": ["Shakked Noy", "Whitney Zhang"],
       "year": 2023,
       "doi": "10.1126/science.adh2586",
       "relevance": "RCT studying how LLM assistance impacts writing-based tasks, key evidence on AI productivity effects and inequality."
     },
     {
       "title": "Generative AI enhances individual creativity but reduces the collective diversity of novel content",
       "authors": ["Anil R Doshi", "Oliver P Hauser"],
       "year": 2024,
       "doi": "10.1126/sciadv.adn5290",
       "relevance": "Experimental study showing AI increases individual creativity but reduces population-level diversity, relevant to collective effects of AI assistance."
     },
     {
       "title": "AI Meets the Classroom: When Do Large Language Models Harm Learning?",
       "authors": ["Matthias Lehmann", "Philipp B. Cornelius", "Fabian J. Sting"],
       "year": 2024,
       "arxiv_id": "2409.09047",
       "relevance": "Lab study on GPT access in programming courses finding students asked for solutions more than explanations, directly relevant to AI in education."
     },
     {
       "title": "Artificial Intelligence in the Knowledge Economy",
       "authors": ["Enrique Ide", "Eduard Talamas"],
       "year": 2025,
       "doi": "10.1086/737233",
       "relevance": "Closest theoretical prior work modeling how AI impacts labor markets with exogenous worker ability, which this paper extends to endogenous investment."
     },
     {
       "title": "Emergent Abilities of Large Language Models",
       "authors": ["Jason Wei", "Yi Tay", "Rishi Bommasani"],
       "year": 2022,
       "doi": "10.48550/arXiv.2206.07682",
       "relevance": "Foundational paper documenting emergent abilities in LLMs, one of the two stylized facts motivating this paper's model."
     },
     {
       "title": "Are Emergent Abilities of Large Language Models a Mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "doi": "10.48550/arXiv.2304.15004",
       "relevance": "Challenges the emergence hypothesis by arguing discontinuities may come from benchmarks rather than models, directly discussed in Section 3.2."
     },
     {
       "title": "Effects of AI Feedback on Learning, the Skill Gap, and Intellectual Diversity",
       "authors": ["Christoph Riedl", "Eric Bogert"],
       "year": 2024,
       "arxiv_id": "2409.18660",
       "relevance": "Studies how AI chess engines impact player learning, finding increased inequality — relevant to AI in education and skill development."
     },
     {
       "title": "The ABC's of Who Benefits from Working with AI: Ability, Beliefs, and Calibration",
       "authors": ["Andrew Caplin", "David Deming", "Shangwen Li", "Daniel Martin", "Philip Marx", "Ben Weidmann", "Kadachi Jiada Ye"],
       "year": 2024,
       "doi": "10.3386/w33021",
       "relevance": "Experimental study on how AI aids human decision-makers, finding that miscalibrated humans accrue less benefit — relevant to the misspecification model in this paper."
     }
   ]
 }