ai-research-survey

scan.json (23643B)

---

 {
   "paper": {
     "title": "How AI Impacts Skill Formation",
     "authors": ["Judy Hanwen Shen", "Alex Tamkin"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.20245",
     "doi": "10.48550/arXiv.2601.20245"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["rct", "qualitative"],
   "key_findings": "In a randomized experiment with 52 professional developers learning a new Python library (Trio), AI assistance reduced quiz scores by 17% (Cohen's d=0.738, p=0.010) without significantly improving task completion time. Qualitative analysis of screen recordings identified six AI interaction patterns: three low-scoring patterns involving cognitive offloading (delegation, progressive reliance, iterative debugging) and three high-scoring patterns involving cognitive engagement (conceptual inquiry, hybrid code-explanation, generation-then-comprehension). The control group encountered more errors, particularly Trio-related errors, which likely contributed to deeper skill formation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Annotated transcripts released at https://github.com/safety-research/how-ai-impacts-skill-formation (Section 6, Appendix B.2)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Annotated transcripts of all 51 participants made publicly available at the GitHub repository (Appendix B.2)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is provided. The coding platform is described but not the computational environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The study design is described in detail but there are no scripts or README for replicating the experiment."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% CI error bars are shown in Figures 5, 6, and 7. Figure captions state 'Error bars represent 95% CI.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "P-values reported for main effects: quiz score p=0.010, task time p=0.391 (Section 5.2.2). Cohen's d reported as effect sizes."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cohen's d reported for main study (d=0.738 for quiz score, d=0.725 controlling for warm-up) and pilot D (d=1.11 task time, d=1.7 quiz score). Percentage differences also provided (17% score difference)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Power analysis conducted based on pilot study D. 'We assumed a conservative effect size of d=0.85 (half of the observed learning effect) to account for the potential effect size inflation typical in pilot studies' (Section 5.1)."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main outcome measures (quiz score, task time) report 95% CIs but not standard deviations. Table 4 reports median and IQR for error counts, but primary outcomes lack explicit variance measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Control group (No AI, n=26) serves as the baseline, completing the same tasks without AI assistance."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The no-AI control condition is the natural and appropriate baseline for measuring the effect of AI assistance on learning."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an RCT comparing two conditions (AI vs no AI), not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two primary metrics: task completion time and quiz score. Quiz score further decomposed into conceptual, debugging, and code reading subareas (Figure 8)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a human subjects study measuring human learning outcomes, not a system whose outputs require human evaluation."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not a machine learning evaluation; no train/test split concept applies."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 8 provides per-task and per-skill-type breakdowns (Task 1, Task 2, Conceptual, Debugging, Code Reading). Figure 7 breaks results by years of coding experience."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 provides detailed qualitative analysis of low-scoring AI interaction patterns (AI Delegation, Progressive AI Reliance, Iterative AI Debugging) and discusses why they lead to poor outcomes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The null result on task completion time (p=0.391, no significant speedup from AI) is a negative result contrary to prior work and is reported prominently."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that AI 'impairs conceptual understanding, code reading, and debugging abilities, without delivering significant efficiency gains' are supported by Figure 6 (quiz score p=0.010, task time p=0.391) and Figure 8 (per-skill breakdown)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are justified by randomized controlled trial design with balanced groups (Table 1), pre-registration, and multiple pilot studies addressing compliance threats."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'How AI Impacts Skill Formation' is very broad. Results are from one specific library (Trio), one AI model (GPT-4o), one task type (learning a new library), and crowd-sourced developers. While Section 7.1 lists limitations, the title and abstract frame findings more broadly than the evidence supports."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses heterogeneity in AI usage patterns as explanation for the null productivity result. Section 7 discusses cognitive engagement as the mechanism. Section 7.1 discusses alternative evaluation designs, task selection, and participant realism."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly frames quiz score as measuring 'skill formation' and discusses what the quiz does and does not capture (Section 4.2). Section 7.1 acknowledges evaluation design as a limitation and suggests alternative measurement strategies."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'The base model used for this assistant is GPT-4o' (Section 4.1) without specifying a version date or API snapshot. Per schema guidance, marketing names without snapshot dates do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'the model is prompted to be an intelligent coding assistant' and 'The AI assistant has access to participants' current version of the code' (Section 4.1) but does not provide the actual system prompt text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other API parameters are reported for the GPT-4o assistant."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The AI assistant is a simple chat interface on a coding platform."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data collection pipeline well documented: recruitment survey → warm-up task → coding task → quiz → post-survey. Disqualification criteria pre-registered (1 participant excluded for leaving 4 blank questions, Section 5.2.1). Pilot study iterations documented in Table 2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 'Future Work' contains substantive discussion of limitations including task selection, task length, participant realism, prompting skills, and evaluation design."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 lists specific threats: single task with chat-based interface (not agentic), one-hour timeframe vs months/years of real skill formation, crowd workers vs actual employees with job incentives, and unmeasured prompting skill differences."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 explicitly states what was NOT tested: agentic coding tools, longitudinal skill development, real company settings, differences in prompting fluency, and alternative evaluation strategies."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Annotated transcripts of all participants made publicly available at https://github.com/safety-research/how-ai-impacts-skill-formation (Appendix B.2)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection thoroughly described: coding platform keystrokes, AI chat transcripts, screen recordings, Google Forms surveys, recruitment surveys (Sections 4.3, Appendix B.1). Instruments and timing documented."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Participants recruited through 'a third party crowd-worker platform' with specific criteria: 1+ years Python experience, weekly Python use, prior AI tool experience, no Trio experience (Section 4.3). Balanced on experience and asyncio familiarity."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented: 58 recruited → 53 completed all parts → 1 disqualified per pre-registered criteria → 52 analyzed (Section 5.2.1). One screen recording unavailable (footnote 3)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Anthropic Fellows Program and Anthropic affiliation disclosed in author footnotes: 'Work done as a part of the Anthropic Fellows Program' and author affiliations."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors' Anthropic affiliations clearly stated: Shen via the Anthropic Fellows Program, Tamkin directly at Anthropic."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Anthropic is a major AI company with strategic interest in how AI assistance is perceived. The study's findings (AI impairs learning if used without cognitive engagement) align with Anthropic's safety-oriented positioning. The funder has a stake in the narrative around responsible AI use."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is an RCT studying human learning outcomes, not evaluating a pre-trained model's capability on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not evaluating a pre-trained model on a benchmark. The AI assistant is a tool used by participants, not the subject of capability evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not evaluating a pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": true,
         "justification": "Study pre-registered at https://osf.io/w49e7 (Section 5.1). 'We submitted the grading rubric for the quiz in our study pre-registration before running the experiment' (Section 4.2)."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "'The protocol was reviewed and approved by internal reviewers at Anthropic' (Appendix A.1). While this is internal review rather than an independent IRB, ethics review and approval is explicitly mentioned."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 17 reports age, education, coding context, and student status. Table 1 reports coding experience, Python frequency, asyncio experience, and async quiz scores by condition."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3: 'We only recruited participants who self-reported having more than one year of Python experience, code in Python at least once a week, have tried AI coding assistance at least a few times, and have never used the Trio library before.'"
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Between-subjects randomized experiment with balanced assignment across coding experience, Python frequency, asyncio familiarity, and pre-task quiz score (Table 1, Section 4.3)."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of blinding. Participants clearly knew their condition: treatment group was prompted to use AI (Figure 24), control group pledged not to use AI (Figure 21). No mention of evaluator blinding for quiz grading or qualitative analysis."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "58 recruited, 53 completed all parts, 1 disqualified for leaving blank questions due to not realizing quiz had multiple parts, 52 analyzed (Section 5.2.1). 4 control participants did not complete task 2 within time limit."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a human subjects RCT studying learning outcomes, not proposing an AI method whose inference cost matters."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a human subjects RCT, not a compute-intensive AI method."
       }
     }
   },
   "claims": [
     {
       "claim": "AI assistance reduces skill formation (quiz scores) by 17% or 2 grade points when learning a new programming library.",
       "evidence": "Main study (n=52): Cohen's d=0.738, p=0.010, 4.15 point difference on 27-point quiz. Effect persists when controlling for warm-up task time (d=0.725, p=0.016). Figures 6, 7.",
       "supported": "strong"
     },
     {
       "claim": "AI assistance does not significantly improve task completion time when new skills are required.",
       "evidence": "Main study: p=0.391 for task time difference between AI and no-AI groups (Figure 6). Qualitative analysis shows time spent interacting with AI offsets potential speedup.",
       "supported": "strong"
     },
     {
       "claim": "Three AI interaction patterns (Conceptual Inquiry, Hybrid Code-Explanation, Generation-Then-Comprehension) preserve learning outcomes by maintaining cognitive engagement.",
       "evidence": "Qualitative analysis of screen recordings (Section 6). High-scoring patterns achieved 65-86% quiz scores vs 24-39% for low-scoring patterns (Figure 11). Small cluster sizes (n=2-7 per pattern).",
       "supported": "moderate"
     },
     {
       "claim": "Debugging skills show the largest gap between AI and non-AI groups.",
       "evidence": "Figure 8 shows debugging questions have the largest score difference between conditions, while code reading has the smallest. Attributed to control group encountering more errors (Table 4).",
       "supported": "moderate"
     },
     {
       "claim": "Participants who fully delegated coding to AI finished fastest but learned the least.",
       "evidence": "AI Delegation cluster (n=4): 19.5 min completion, 39% quiz score vs control group 23 min and higher scores (Figure 11, Section 6).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company conflict of interest",
       "detail": "Both authors are affiliated with Anthropic (a major AI company). The finding that AI impairs learning without cognitive engagement aligns with Anthropic's safety-oriented messaging. No independent competing interests statement is provided."
     },
     {
       "flag": "Small subgroup sizes in qualitative analysis",
       "detail": "The six AI interaction patterns have cluster sizes of n=2 to n=7. Quantitative conclusions about these patterns (quiz scores, completion times) are drawn from very small groups."
     },
     {
       "flag": "Internal ethics review only",
       "detail": "Ethics review was conducted by 'internal reviewers at Anthropic' rather than an independent IRB, which is unusual for human subjects research and provides less oversight."
     },
     {
       "flag": "AI model setup underspecified",
       "detail": "GPT-4o version, system prompt, and hyperparameters are not reported. The AI assistant's behavior could significantly affect results, and this setup is not reproducible."
     },
     {
       "flag": "Short task duration may not reflect real skill formation",
       "detail": "The entire coding task is 35 minutes. Real skill formation occurs over months to years. The authors acknowledge this in Section 7.1 but the framing ('How AI Impacts Skill Formation') suggests broader applicability."
     }
   ],
   "cited_papers": [
     {
       "title": "The impact of ai on developer productivity: Evidence from github copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Key prior work on AI coding productivity gains (55.5% speedup claim), directly contrasted with this paper's null productivity result."
     },
     {
       "title": "The effects of generative ai on high skilled work: Evidence from three field experiments with software developers",
       "authors": ["Zheyuan Kevin Cui", "Mert Demirer", "Sonia Jaffe", "Leon Musolff", "Sida Peng", "Tobias Salz"],
       "year": 2024,
       "relevance": "Field experiments showing 26.8% productivity boost from AI code completions, with less experienced coders benefiting more."
     },
     {
       "title": "Navigating the jagged technological frontier: Field experimental evidence of the effects of ai on knowledge worker productivity and quality",
       "authors": ["Fabrizio Dell'Acqua", "Edward McFowland III", "Ethan R Mollick"],
       "year": 2023,
       "relevance": "Influential field experiment on AI-enhanced consulting productivity showing 12.2% more task completions."
     },
     {
       "title": "Generative ai at work",
       "authors": ["Erik Brynjolfsson", "Danielle Li", "Lindsey Raymond"],
       "year": 2025,
       "relevance": "AI call center study showing 15% productivity improvement, with less experienced workers benefiting most."
     },
     {
       "title": "Measuring the impact of early-2025 ai on experienced open-source developer productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "RCT finding slowdown effects for expert coders, supporting this paper's finding that AI does not always improve productivity."
     },
     {
       "title": "Which economic tasks are performed with ai? evidence from millions of claude conversations",
       "authors": ["Kunal Handa", "Alex Tamkin", "Miles McCain"],
       "year": 2025,
       "arxiv_id": "2503.04761",
       "relevance": "Large-scale analysis of real-world AI usage patterns across professional domains including software engineering."
     },
     {
       "title": "Measuring progress on scalable oversight for large language models",
       "authors": ["Samuel R Bowman", "Jeeyoon Hyun", "Ethan Perez"],
       "year": 2022,
       "arxiv_id": "2211.03540",
       "relevance": "Foundational work on the scalable oversight problem — supervising increasingly capable AI systems requires maintaining human competence."
     },
     {
       "title": "GenAI as an exoskeleton: Experimental evidence on knowledge workers using genai on new skills",
       "authors": ["Emma Wiles", "Lisa Krayer", "Mohamed Abbadi"],
       "year": 2024,
       "relevance": "Directly related finding that AI-enhanced technical abilities did not persist when workers lost AI access."
     },
     {
       "title": "Experimental evidence on the productivity effects of generative artificial intelligence",
       "authors": ["Shakked Noy", "Whitney Zhang"],
       "year": 2023,
       "relevance": "Early experimental evidence on AI writing productivity effects, finding lower-skilled workers benefit most."
     },
     {
       "title": "Rocks coding, not development: A human-centric, experimental evaluation of llm-supported se tasks",
       "authors": ["Wei Wang", "Huilong Ning", "Gaowei Zhang"],
       "year": 2024,
       "relevance": "Human-centric evaluation of LLM-supported software engineering tasks with diverse interaction pattern analysis."
     },
     {
       "title": "The impact of generative ai on critical thinking: Self-reported reductions in cognitive effort and confidence effects from a survey of knowledge workers",
       "authors": ["Hao-Ping Lee", "Advait Sarkar", "Lev Tankelevitch"],
       "year": 2025,
       "relevance": "Survey finding that generative AI reduces cognitive effort and confidence in knowledge workers, supporting cognitive offloading concerns."
     }
   ]
 }