ai-research-survey

scan.json (23410B)

---

 {
   "paper": {
     "title": "Not Everyone Wins with LLMs: Behavioral Patterns and Pedagogical Implications for AI Literacy in Programmatic Data Science",
     "authors": ["Qianou Ma", "Kenneth Koedinger", "Tongshuang Wu"],
     "year": 2025,
     "venue": "CHI 2026",
     "arxiv_id": "2509.21890",
     "doi": "10.1145/3772318.3791283"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["observational", "qualitative"],
   "key_findings": "Technical experience (programming/data science) remains a significant predictor of success even with LLM access (β=6.09, p=.041), while self-reported LLM experience and communication skills do not predict performance. Under time pressure, LLMs may temporarily close the experience gap, but given ample time, technically experienced students outperform novices. Experienced students use AI more strategically (proactive planning, clearer prompts) while novices use it reactively for debugging. Lightweight demonstrations improve prompting quality but evaluation skills remain a persistent bottleneck.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we open-source our code: https://github.com/mqo00/dspm' in Section 1 contributions."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset release mentioned. The homework logs, surveys, and screen recordings are not made available, likely due to student privacy."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification or dependency requirements mentioned. The LLM annotation uses claude-sonnet-4-20250514 with temperature=0 (Appendix E) but no broader environment setup is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions provided. The open-sourced code repository is mentioned but no README or reproduction guide is described in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The regression results (Table 3) report estimates and standard errors but no confidence intervals. Figures show distributions without error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Mann-Whitney U test (p=0.027) for HW0 experience gap, and linear mixed-effects regression with p-values reported (Table 3, p=.041 for technical experience)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Regression coefficients (β=6.09) with standard errors are reported in Table 3, and Cohen's kappa κ=0.71 for inter-rater reliability. These provide magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "N=36 students with no power analysis or justification. The paper acknowledges in Section 8.2 that 'it is a small sample that constrained our ability to derive conclusive subgroup analysis' but does not justify why 36 was sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Standard errors for regression coefficients are reported but no variance across experimental conditions or runs. The LLM annotation was run with temperature=0 so single-run only."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "HW0 includes a within-subjects baseline: half the class completed tasks without AI and half with AI, enabling comparison. Post-demo behaviors are compared to pre-demo as a baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The comparisons are within the study design (with/without AI, pre/post instruction, experienced vs novice). These are appropriate contemporaneous comparisons for this study type."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational classroom study, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: homework grades (rubric-based), AI use frequency, step success rates, appropriate AI use ratios, and qualitative behavioral codes."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Homework was graded by two TAs with inter-rater reliability measured (κ=0.71). Screen recordings and logs were manually analyzed by the authors for codebook development."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not a prediction/classification study. No train/test split is applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by experience level (experienced vs novice), by AI code type, by episode step, and by pre/post instruction (Figures 4-8)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides detailed failure case comparisons between experienced and novice behaviors. Multiple failure patterns are discussed including 'prompting rabbit holes' and repeated KeyError loops."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key negative result: LLMs do NOT close the experience gap when given ample time. Also: evaluation behaviors remained a bottleneck even after instruction, and LLM/communication experience did not predict grades."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about technical experience predicting success (supported by Table 3), behavioral differences (supported by Figures 4-6), and instruction limitations (supported by Figures 7-8) are all backed by results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses causal language like 'time constraints may remove the performance gap brought by prior technical experience' and 'LLMs do not fully close the experience gap.' However, the observational study design (no randomization of experience levels, confounded pre/post comparison with time-on-task) does not support causal inference. The paper partially acknowledges this in Section 8.2."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.2 explicitly bounds generalization: small sample (N=36), single course at one R1 university, specific tool (Colab Gemini), tasks that were 'not LLM-hard.' The paper calls for extension to 'more diverse task contexts.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.2 discusses alternative explanations: correlated experience dimensions (Python and data science ρ=0.52), time-on-task confound in pre/post comparison, self-report bias (Dunning-Kruger), and undisclosed external AI use."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses rubric-graded homework scores as the outcome measure but frames findings broadly in terms of 'success' and 'performance' without discussing what these grades capture vs. real-world AI-assisted data analysis competence. The gap between classroom grades and authentic AI use effectiveness is not explicitly acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The Gemini model used in Colab is explicitly stated as unknown: 'Specific model used by Google Colab in Spring 2025 is unknown, but it could be gemini-2.0-flash or gemini-1.5-flash.' The annotation model claude-sonnet-4-20250514 is specified with version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts for the LLM-based log annotation are provided in Appendix E, including segmentation and step-wise annotation prompts with complete instructions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "For the LLM annotation: 'temperature of 0 is used with the model claude-sonnet-4-20250514 and max_tokens of 20000' (Appendix E)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The study observes students using Colab Gemini (a third-party tool evaluated as a black box)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes data filtering criteria: excluding students who used other AI tools, those who disengaged, log length filtering, and the resulting sample (7,315 events from 28 students). Appendix B provides detailed data source information per analysis method."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.2 'Limitations and Future Work' provides substantive discussion of limitations spanning multiple paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.2 discusses specific threats: small sample (N=36), correlated experience measures, self-report bias citing Kruger & Dunning, undisclosed external AI tool use, Colab's changing interface, and the confound between instruction and time-on-task."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.2 explicitly states boundaries: tasks were 'not LLM-hard,' single graduate course, specific tool (Colab Gemini), logging limitations compared to established tools like JELAI. The paper calls for extending to 'more complex tasks' and 'more diverse task contexts.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw log data, survey responses, and screen recordings are not made available, likely due to student privacy. Only code is open-sourced."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes data collection in detail: custom browser script intercepting Colab API requests, localStorage capture, JSON export, survey instruments at multiple time points, and screen recordings. Appendix C provides full survey questions."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the context: graduate-level DSPM course at an R1 university with 41 students (36 completing). Section 4.1 characterizes the diverse backgrounds (finance, journalism, physics, music, CS). Recruitment is via course enrollment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: raw logs → episode segmentation → step-wise annotation (Section 4.3), with filtering criteria and resulting counts (16,315 total events → 7,315 annotated events from 28 students, Appendix B)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists NSF awards CNS-2213791 and 2414915, and Google Academic Research Award."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as Carnegie Mellon University. The study uses Google Colab/Gemini but the authors are not Google employees."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Google Academic Research Award is listed as a funder, and the study evaluates Google's Colab Gemini product. Google has a financial interest in Gemini being perceived positively, creating a potential conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This study does not evaluate a pre-trained model's capability on a benchmark. It observes how students use an LLM tool in coursework."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation study. The study observes human behavior with LLM tools."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration (OSF, AsPredicted, or similar) anywhere in the paper."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1: 'Our IRB-approved study was conducted in Spring 2025.'"
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes students from 'finance, telecommunications, journalism, physics, music, and computer science.' Figure 2 shows distribution of experience levels. Appendix C includes demographic questions (age, gender). Pre-survey collected backgrounds."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes filtering criteria for log analysis: excluded students who used external AI tools, those who disengaged (log length below median). Section 4.1 notes 41 started, 36 completed HW4. Appendix B details per-analysis sample sizes."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "For HW0: 'half the class completed the first two or last two tasks without using GenAI' (Section 3.2), providing counterbalanced conditions. The paper states this was 'counterbalanced across two tasks.'"
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No mention of blinding. Students knew whether they had AI access. Graders (TAs) are not described as blinded to student experience level or AI condition."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: '41 students completed HW0, with a slight attrition to 36 students by HW4.' Section 4.3 further notes filtering from 36 to 28 students for log analysis with reasons."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a classroom observational study, not proposing a computational method. Cost is irrelevant."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a classroom observational study, not a compute-intensive method."
       }
     }
   },
   "claims": [
     {
       "claim": "Technical experience significantly predicts homework performance even with LLM access, while LLM experience and communication skills do not.",
       "evidence": "Linear mixed-effects regression (Table 3): technical β=6.09, p=.041; LLM β=-3.05, p=.266; communication β=-2.54, p=.381. N=36 students, HW1-4.",
       "supported": "moderate"
     },
     {
       "claim": "Under time pressure (HW0), LLMs close the technical experience gap, but the gap persists with ample time (HW1-4).",
       "evidence": "Mann-Whitney U test: significant gap without LLM (p=0.027), not significant with LLM (p>0.05) in timed HW0. Regression on untimed HW1-4 shows persistent gap (Section 5).",
       "supported": "moderate"
     },
     {
       "claim": "Experienced students use AI more strategically while novices use it reactively.",
       "evidence": "Figure 5: experienced students self-resolve challenges more often; when using AI, experienced ask for explanation more (75% vs 57% for challenges) while novices use AI for planning more tied to challenges (64% vs 43%). Figure 6: experienced students have higher AI coding success rate (90% vs 79%) and more prompt quality attributes.",
       "supported": "moderate"
     },
     {
       "claim": "Lightweight demonstrations improve prompting quality but evaluation skills remain a bottleneck.",
       "evidence": "Figure 8: post-demo appropriate AI use ratios for prompt quality improved ~30%. However, evaluation behaviors (ai_critique_output, explain_code, explain_error) remained below 50% post-demo. Figure 7 shows more students reach later workflow steps post-demo.",
       "supported": "moderate"
     },
     {
       "claim": "LLM-based log annotation achieves 75% accuracy for behavior coding.",
       "evidence": "Section 4.3: 'An author verified 10% of the log annotations, and LLM achieved an accuracy of 75% (specifically, 81.3% accuracy for step success, 76.7% for encountered challenge, and 70.9% for AI use behaviors annotations).'",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Small sample size for quantitative claims",
       "detail": "N=36 students (further filtered to 28 for log analysis) is small for regression analysis with three predictors. The paper acknowledges this but still draws strong conclusions. The Mann-Whitney U test finding (p=0.027 vs p>0.05) could easily be a power issue rather than a real interaction."
     },
     {
       "flag": "Confounded pre/post comparison",
       "detail": "The pre-post instruction comparison (RQ3) is confounded with time-on-task (15-minute timed HW0 vs bi-weekly assignments), different datasets, and practice effects. The paper acknowledges this but still attributes improvements to instruction."
     },
     {
       "flag": "LLM-based annotation at 75% accuracy used for quantitative analysis",
       "detail": "The automated log annotation (claude-sonnet-4-20250514) achieved only 75% overall accuracy (70.9% for AI use behaviors), and only 10% was verified. This introduces substantial noise into the quantitative behavioral analyses in Sections 6 and 7."
     },
     {
       "flag": "Unknown LLM model version for the intervention tool",
       "detail": "The Gemini model students used is unknown ('could be gemini-2.0-flash or gemini-1.5-flash'). Different model capabilities would affect all findings about AI effectiveness."
     },
     {
       "flag": "Google funding + Google product evaluation",
       "detail": "The study is partially funded by a Google Academic Research Award and evaluates Google's Colab Gemini product. This potential conflict is not acknowledged."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring the impact of early-2025 AI on experienced open-source developer productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "RCT measuring AI impact on developer productivity, finding expert performance may be hindered by AI."
     },
     {
       "title": "Learning agent-based modeling with LLM companions: Experiences of novices and experts using ChatGPT & NetLogo chat",
       "authors": ["John Chen", "Xi Lu", "Yuzhou Du"],
       "year": 2024,
       "doi": "10.1145/3613904.3642377",
       "relevance": "Studies novice vs expert differences in AI-assisted programming, finding experts benefit more from LLM companions."
     },
     {
       "title": "Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming",
       "authors": ["Majeed Kazemitabaar"],
       "year": 2023,
       "doi": "10.1145/3544548.3580919",
       "relevance": "Studies how AI code generators affect novice programming learning outcomes."
     },
     {
       "title": "How much does AI impact development speed? An enterprise-based randomized controlled trial",
       "authors": ["Elise Paradis"],
       "year": 2025,
       "relevance": "Enterprise RCT of AI impact on development speed with mixed evidence on productivity gains."
     },
     {
       "title": "The impact of ai on developer productivity: Evidence from github copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "relevance": "Foundational study on GitHub Copilot's productivity impact on developers."
     },
     {
       "title": "How Beginning Programmers and Code LLMs (Mis)read Each Other",
       "authors": ["Sydney Nguyen"],
       "year": 2024,
       "doi": "10.1145/3613904.3642706",
       "relevance": "Studies how novice programmers interact with and misunderstand LLM-generated code."
     },
     {
       "title": "What should we engineer in prompts? Training humans in requirement-driven LLM use",
       "authors": ["Qianou Ma"],
       "year": 2025,
       "doi": "10.1145/3731756",
       "relevance": "Studies prompt engineering education and requirement specification for LLM interactions."
     },
     {
       "title": "How humans communicate programming tasks in natural language and implications for end-user programming with LLMs",
       "authors": ["Madison Pickering"],
       "year": 2025,
       "doi": "10.1145/3706598.3713271",
       "relevance": "Studies how humans express programming tasks to LLMs and whether prior experience matters."
     },
     {
       "title": "LLMs are imperfect, then what? An empirical study on LLM failures in software engineering",
       "authors": ["Jiessie Tie"],
       "year": 2024,
       "arxiv_id": "2411.09916",
       "relevance": "Documents LLM failure patterns in software engineering including 'prompting rabbit holes.'"
     },
     {
       "title": "The impact of generative AI on collaborative open-source software development: Evidence from GitHub Copilot",
       "authors": ["Fangchen Song", "Ashish Agarwal", "Wen Wen"],
       "year": 2024,
       "relevance": "Studies Copilot's impact on collaborative open-source development with mixed findings."
     }
   ]
 }