ai-research-survey

scan.json (24672B)

---

 {
   "scan_version": 2,
   "active_modules": [],
   "paper": {
     "title": "Not Everyone Wins with LLMs: Behavioral Patterns and Pedagogical Implications for AI Literacy in Programmatic Data Science",
     "authors": ["Qianou Ma", "Kenneth Koedinger", "Tongshuang Wu"],
     "year": 2026,
     "venue": "CHI 2026",
     "doi": "10.1145/3772318.3791283",
     "arxiv_id": "2509.21890"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we open-source our code: https://github.com/mqo00/dspm' in Section 1."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not mention releasing the student log data, survey responses, or homework grades. The GitHub link is for code only. The datasets used in homework (car and game datasets) are not explicitly released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is mentioned. The paper mentions Google Colab and Python but provides no dependency details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The study involves a specific classroom setting that cannot be trivially reproduced, and no instructions for replicating the analysis pipeline are given."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The regression results in Table 3 report estimates and standard errors but no confidence intervals. No error bars are mentioned on figures. The Mann-Whitney U test reports only p-values."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Mann-Whitney U tests (Section 5, p=0.027) and linear mixed-effects regression with p-values (Table 3, p=.041 for technical experience). Cohen's kappa is used for inter-rater reliability (κ=0.71)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The regression coefficient β=6.09 is reported in Table 3 with standard errors, providing effect size context. Cohen's kappa κ=0.71 is an effect size measure for agreement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample size of 36 students is not justified with a power analysis. The paper acknowledges in Section 8.2 that 'it is a small sample that constrained our ability to derive conclusive subgroup analysis' but does not justify why 36 was sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Standard errors are reported for regression coefficients in Table 3, but no variance or standard deviation is reported across experimental conditions or student performance distributions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "HW0 includes a within-subjects baseline: half the class completed tasks without AI and half with AI (counterbalanced), enabling comparison of AI vs. no-AI performance (Section 3.2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline is the same students working without AI, which is a direct and appropriate comparison for the research question. The paper also references contemporary prior work."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study of student behaviors, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: homework grades (rubric-based), AI usage frequency, step success rate, appropriate AI use ratio, and qualitative behavioral codes across multiple dimensions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Homework submissions were graded by two TAs with inter-rater reliability assessment (κ=0.71). An author manually verified 10% of LLM annotations achieving 75% accuracy (Section 4.3)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a benchmark evaluation; it's a classroom study. There is no train/test split applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by experience level (experienced vs. novice), by AI usage code type (improve, code, explain, evaluate, prompt), by episode step (intent, input, understand, assess), and pre/post instruction (Figures 4-8)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides detailed comparison of failure behaviors between novices and experienced students. Section 7 discusses specific failure patterns like repeated KeyError loops and early stops."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that LLM and communication experience did NOT predict grades (Table 3, p=.266 and p=.381). It also reports that evaluation behaviors remained a bottleneck even after instruction (Section 7)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: technical experience predicts success (Table 3), students vary in AI leverage across stages (Figures 4-6), and success/failure behaviors are identified (Table 2, Table 5). The claim about lightweight demonstrations being insufficient is supported by Section 7 analysis."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-adjacent claims such as 'LLMs may offset technical experience's advantages' and 'technical experience significantly predicted higher grades.' The study design is observational with self-reported experience, not randomized, and cannot support causal claims. Confounds between experience types are acknowledged but not resolved."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.2 explicitly bounds generalization: 'small sample,' 'authentic classroom contexts,' and notes the tasks 'were not LLM-hard.' The paper calls for extending to 'more diverse task contexts' and notes the specific tool (Colab Gemini) limits generalizability."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.2 discusses multiple alternative explanations: time-on-task vs. instruction effects, correlated experience dimensions (Python and data science, ρ=0.52), potential external AI use, and Dunning-Kruger effects in self-reported experience."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses rubric-based homework grades as a measure of performance and discusses limitations of self-reported experience measures. It acknowledges that self-reported LLM experience 'may not straightforwardly transfer to complex programming data science tasks' (Section 5)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Specific model used by Google Colab in Spring 2025 is unknown, but it could be gemini-2.0-flash or gemini-1.5-flash' (footnote 1). For log annotation, it specifies 'claude-sonnet-4-20250514' which is a versioned model. However, the primary experimental model (Gemini) is unspecified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E provides the full prompts used for LLM-assisted log annotation, including segmentation and step-wise annotation prompts with complete text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 and Appendix E report 'temperature=0 for deterministic generations' and 'max_tokens of 20000' for the log annotation model."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The study examines students using Colab's built-in Gemini assistant, which is a third-party tool evaluated as a black box."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 documents data filtering: 'filtered out students who self-reported other tool use like ChatGPT,' filtered by log length relative to median, and describes the annotation pipeline with segmentation and step-wise coding. The filtering from 41 to 36 students is explained (Section 4.1), and further filtering to 28 students for log analysis is documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.2 'Limitations and Future Work' provides substantive discussion across two subsections: 'Going deeper' and 'Going broader.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.2 discusses study-specific threats: small sample (36 students), correlated experience dimensions preventing independent analysis, self-report bias (citing Dunning-Kruger), undisclosed external AI use, and reliance on custom logging script that 'can be less robust than existing Jupyter notebook logging toolkits.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.2 explicitly states the tasks 'were not LLM-hard,' the study is limited to Google Colab with Gemini, and calls for extension to 'more diverse task contexts.' It notes 'we did not experimentally isolate the effect for time-on-task.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw log data, survey responses, grades, and screen recordings are not made available. Only analysis code is open-sourced."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 provides detailed description of all data sources: notebooks/grades, behavioral logs (custom browser script intercepting Colab endpoints), surveys (pre/post/final with questions in Appendix C), and screen recordings. Total: 16,315 logged events and 44 hours of recordings."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes participants as students in a specific graduate-level course (DSPM) at an R1 university (CMU). The course's diverse cohort is described, including product managers, engineers, and non-technical graduates. This is a convenience sample from an existing class."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: raw logs → segmentation into episodes → step-wise annotation → validation (10% manual check, 75% accuracy). Filtering steps are described with counts: 41 students → 36 after attrition, further to 28 after quality filtering. 7,315 events annotated from the filtered set."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The acknowledgments section discloses funding: 'National Science Foundation (award CNS-2213791, 2414915) and Google Academic Research Award.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are disclosed as Carnegie Mellon University affiliates. The study uses Google Colab with Gemini, and funding includes a Google Academic Research Award, but the paper does not evaluate Google's product favorably — it actually documents many limitations of the Gemini assistant."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Google Academic Research Award is a funder, and the study uses Google's Colab with Gemini assistant. Google has a financial interest in positive perceptions of its AI tools, though the paper's findings are not particularly favorable to the tool."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on a benchmark. It studies human behavior when using an LLM tool."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of model capability is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of model capability is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration (OSF, AsPredicted, or other registry) is found in the paper."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states 'Our IRB-approved study was conducted in Spring 2025.'"
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports students' fields (finance, telecommunications, journalism, physics, music, CS), experience distributions (Figure 2), and the pre-survey collected demographics (age, gender per Appendix C). Self-reported experience levels across technical, LLM, and communication dimensions are detailed."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes filtering criteria for log analysis: excluded students who self-reported external AI tool use, those showing external tool use in recordings, and those with below-median log length. The broader study included all 36 course completers from the initial 41."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes HW0 randomization: 'half the class completed the first two or last two tasks without using GenAI,' providing a counterbalanced within-subjects design."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described. Students knew whether they were in the AI or no-AI condition for HW0. TAs grading homework are not described as blinded to student experience levels or AI usage."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports attrition: '41 students completed HW0, with a slight attrition to 36 students by HW4.' For log analysis, further filtering to 28 students is documented with reasons (Section 4.3)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a classroom study, not a system or method paper. Cost reporting is not applicable."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a classroom study, not a system or method paper. Compute budget is not applicable."
       }
     }
   },
   "claims": [
     {
       "claim": "Technical experience significantly predicts homework performance with LLM access, while LLM experience and communication experience do not.",
       "evidence": "Linear mixed-effects regression (Table 3): technical experience β=6.09, p=.041; LLM experience β=-3.05, p=.266; communication experience β=-2.54, p=.381. N=36 students across HW1-4.",
       "supported": "moderate"
     },
     {
       "claim": "Under time pressure (HW0), LLMs close the performance gap between technically experienced and novice students.",
       "evidence": "Mann-Whitney U test: significant gap without LLMs (p=0.027), gap disappears with LLMs (p>0.05). Section 5.",
       "supported": "moderate"
     },
     {
       "claim": "Experienced students use AI more strategically, asking for explanations when facing challenges (75% vs 57%) and proactively requesting plans, while novices use AI more reactively.",
       "evidence": "Figure 5B shows distribution of AI usage by experience level and challenge encounter status, based on annotated logs from 28 students (Section 6).",
       "supported": "weak"
     },
     {
       "claim": "Experienced students generate code with AI more successfully (90% vs 79% step success rate) and write prompts with more quality attributes.",
       "evidence": "Figure 6A shows step success rates by experience level; Figure 6B shows count of prompt quality attributes by experience and grade. Based on annotated log analysis.",
       "supported": "weak"
     },
     {
       "claim": "Lightweight demonstration improves surface AI fluency (prompt quality +30% appropriate use ratio) but evaluation behaviors remain a bottleneck (<50% post-demo).",
       "evidence": "Figure 8 shows pre-to-post appropriate AI use ratios. Figure 7 shows shift in last successful step distribution. Section 7. Note: confounded with extended time-on-task.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["observational", "qualitative"],
   "key_findings": "In a graduate course with 36 students using Google Colab's Gemini assistant for data analysis, technical experience (not LLM familiarity or communication skills) remained the strongest predictor of homework performance. Under time pressure, LLMs appeared to close the experience gap, but with ample time the gap persisted. Experienced students used AI more strategically (clearer prompts, proactive planning), while novices relied on AI reactively. Lightweight demonstration improved prompting behaviors but evaluative skills (critiquing outputs, explaining errors) remained consistently underutilized.",
   "red_flags": [
     {
       "flag": "Small sample size",
       "detail": "N=36 students (further filtered to 28 for log analysis) is small for the quantitative claims being made. Subgroup comparisons (experienced vs. novice) have very limited power, and the regression with 3 predictors on 36 participants is borderline."
     },
     {
       "flag": "Self-reported experience measures",
       "detail": "Experience levels are operationalized via self-report survey items, which the paper itself acknowledges are subject to miscalibration (citing Dunning-Kruger, ref 31). The median split into 'experienced' vs 'novice' is a crude categorization."
     },
     {
       "flag": "Time-on-task confound",
       "detail": "The pre-post comparison (HW0 vs HW1-2) confounds instruction effects with extended time-on-task. The paper acknowledges this ('any improvement pre- to post-demo is a result of both extended time-on-task and the instruction') but still draws conclusions about teachability."
     },
     {
       "flag": "LLM annotation accuracy",
       "detail": "The automated log annotation achieved only 75% accuracy overall (70.9% for AI use behaviors), meaning about 1 in 4 behavioral annotations may be incorrect. Quantitative analyses built on these annotations inherit this error."
     },
     {
       "flag": "Google funding with Google tool evaluation",
       "detail": "The study received a Google Academic Research Award while using Google Colab's Gemini assistant as the primary AI tool. Though findings are not particularly favorable to Google, the conflict is undisclosed."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring the impact of early-2025 AI on experienced open-source developer productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "RCT measuring AI impact on developer productivity, finding experts' performance may be hindered by AI."
     },
     {
       "title": "Learning agent-based modeling with LLM companions: Experiences of novices and experts using ChatGPT & NetLogo chat",
       "authors": ["John Chen", "Xi Lu", "Yuzhou Du"],
       "year": 2024,
       "doi": "10.1145/3613904.3642377",
       "relevance": "Studies novice vs expert differences when collaborating with AI for programming, finding experts benefit more."
     },
     {
       "title": "Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming",
       "authors": ["Majeed Kazemitabaar"],
       "year": 2023,
       "doi": "10.1145/3544548.3580919",
       "relevance": "Studies effect of AI code generators on novice programmers, finding prior experience significantly affects benefit."
     },
     {
       "title": "How much does AI impact development speed? An enterprise-based randomized controlled trial",
       "authors": ["Elise Paradis", "Kate Grey"],
       "year": 2025,
       "relevance": "Enterprise RCT measuring AI impact on development speed with mixed results on productivity."
     },
     {
       "title": "The impact of ai on developer productivity: Evidence from github copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "relevance": "GitHub Copilot productivity study finding less experienced developers benefit more from AI assistants."
     },
     {
       "title": "How Beginning Programmers and Code LLMs (Mis)read Each Other",
       "authors": ["Sydney Nguyen", "Hannah Mclean Babe"],
       "year": 2024,
       "doi": "10.1145/3613904.3642706",
       "relevance": "Studies how beginners interact with code LLMs, finding they do not learn from code feedback and self-experimentation."
     },
     {
       "title": "What should we engineer in prompts? Training humans in requirement-driven LLM use",
       "authors": ["Qianou Ma", "Weirui Peng"],
       "year": 2025,
       "doi": "10.1145/3731756",
       "relevance": "Studies teaching requirement specification for LLM use, relevant to AI literacy and prompt engineering education."
     },
     {
       "title": "The impact of generative AI on collaborative open-source software development: Evidence from GitHub Copilot",
       "authors": ["Fangchen Song", "Ashish Agarwal", "Wen Wen"],
       "year": 2024,
       "relevance": "Studies AI impact on collaborative open-source development, reporting conflicting effects on efficiency."
     },
     {
       "title": "How humans communicate programming tasks in natural language and implications for end-user programming with LLMs",
       "authors": ["Madison Pickering", "Helena Williams"],
       "year": 2025,
       "doi": "10.1145/3706598.3713271",
       "relevance": "Studies how humans express programming tasks to LLMs, finding no significance of prior programming experience in some settings."
     },
     {
       "title": "LLMs are imperfect, then what? An empirical study on LLM failures in software engineering",
       "authors": ["Jiessie Tie", "Bingsheng Yao"],
       "year": 2024,
       "arxiv_id": "2411.09916",
       "relevance": "Empirical study of LLM failure patterns in software engineering including 'prompting rabbit holes.'"
     }
   ]
 }