ai-research-survey

scan.json (26814B)

---

 {
   "paper": {
     "title": "Changes in Coding Behavior and Performance Since the Introduction of LLMs",
     "authors": ["Yufan Zhang", "Jaromir Savelka", "Seth Copen Goldstein", "Michael Conway"],
     "year": 2026,
     "venue": "LAK'26 (The 15th International Learning Analytics and Knowledge Conference)",
     "arxiv_id": "2601.11835",
     "doi": "10.1145/3785022.3785075"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or supplementary material link is provided anywhere in the paper. The analysis scripts for computing edit distances, correlations, and generating figures are not released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The dataset of 2,066 student submissions is not released. No download link or data repository is mentioned. This is understandable given student privacy concerns, but the data is not publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, library versions, or dependency files are mentioned. The paper uses Myers algorithm (git diff default) for edit distances but does not specify the analysis environment or tools used for statistical computations."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The paper describes the metrics (edit distance via Myers algorithm, number of submissions, scores) but does not provide scripts or step-by-step instructions to replicate the analysis."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The figures use box plots which show quartiles, but the paper's main quantitative claims (correlations, percentage changes) are reported as point estimates without confidence intervals. For example, corr=-0.16 and corr=0.240 are reported without CIs."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports p-values for correlations: 'negative correlation between IP Score and Average Edit Distance (corr=-0.16, p<0.001)' and 'moderate, positive correlation (corr=0.240, p<0.001)' in Section 5.3."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports correlation coefficients (corr=-0.16, corr=0.240) as effect sizes and provides contextual magnitude: 'for each additional 60 lines in Average Edit Distance, there is an associated 1% decrease in one's IP Score.' Percentage changes are also given (300% increase in median Average Edit Distance, 500% increase in Total Edit Distance)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample consists of 721 enrollments across 10 semesters, but no justification for this sample size is provided and no power analysis is discussed. The sample size is dictated by the available institutional data rather than statistical considerations."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Box plots in Figures 4-13 show median, quartiles, and outliers for all metrics across semesters. The distributions are visually displayed with spread measures (IQR visible in box plots)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study uses a pre-ChatGPT vs. post-ChatGPT comparison (5 semesters before Fall 2022, 5 semesters after), which serves as the baseline comparison. Pre-ChatGPT semesters are the natural control group."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The pre/post comparison is inherent to the study design. The baseline (pre-ChatGPT semesters Fall 2020 - Spring 2022) is the most appropriate comparison for this quasi-longitudinal design."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study analyzing naturally occurring data, not a system with components that can be ablated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: Number of Submissions, Total Edit Distance, Average Edit Distance, Task Score, IP Score, TP Score, and Average Submission Effect (Sections 4.1, 4.2, and 5.3)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of outputs is not relevant here. The study analyzes automatically recorded student submissions and auto-graded scores. The claims are about behavioral patterns, not about the quality of outputs requiring human judgment."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a predictive modeling study. There is no model being trained or evaluated on a test set. The study analyzes the full population of student submissions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per semester (10 semesters shown individually in Figures 4-13), providing detailed temporal granularity. Pre-ChatGPT and Post-ChatGPT breakdowns are also provided in the correlation analyses (Figures 12-13)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where their analysis is limited: the ceiling effect in Task Score (Section 5.2: 'nearly all students who attempted the PageRank task ultimately received a perfect Task Score'), the disappearance of the negative correlation for TP Score Post-ChatGPT (Section 5.3), and the anomalous f20 COVID semester."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several null or counter-hypothesis results: no discernible trend in IP Score (Section 5.2), TP Scores actually increased post-ChatGPT contrary to what might be expected (Section 5.2), and the disappearance of the negative edit distance-TP score correlation post-ChatGPT (Section 5.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about increased edit distances, more submissions, and decreased average score improvement are supported by the results in Sections 5.1-5.3 and Figures 4-13. The abstract appropriately hedges: 'we cannot definitively attribute them to LLM misuse' and 'suggesting that both student productivity and learning have decreased.'"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper frequently uses causal language ('LLMs are impacting our students,' 'negatively affecting their learning outcomes,' 'has changed how students interact') while acknowledging in the Limitations section that they 'do not know definitively to what extent, or if at all, any student utilized LLMs.' The observational design cannot support causal claims. The paper hedges in some places but makes causal attributions in others, particularly in the Discussion (Section 6) and Conclusion (Section 8)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Changes in Coding Behavior and Performance Since the Introduction of LLMs' is broad, but the study examines only one task (PageRank in Scala) in one course at one university. The Limitations section acknowledges Scala-specificity but the title, abstract, and conclusion generalize to 'students,' 'junior developers,' and workforce impacts without adequate bounding. Section 8 speculates about 'junior developers' producing 'less effective commits' based on one course's data."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section (Section 7) discusses two specific alternative explanations: (1) students sharing solutions across semesters could explain changes, and (2) the inability to confirm LLM usage. The paper also discusses the COVID outlier for f20 and the ceiling effect in Task Scores as confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The study does not use any LLM models in its methodology. It observes student behavior changes coinciding with ChatGPT's release but does not itself use or test any LLM."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The study does not use prompting. It analyzes historical student submission data."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No LLM or machine learning model is used in the study methodology. The study performs statistical analysis of observational data."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is an observational analysis of student submissions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the data processing: 936 total enrollments filtered to 721 who attempted the PageRank task, source code files concatenated (Bash and Scala into single text file per submission), edit distances computed via Myers algorithm (git diff default). The selection criteria for the PageRank task (unchanged over 10 semesters) are explained."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is titled 'Limitations and Future Work' and contains substantive discussion of three specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: (1) inability to confirm LLM usage and students' incentive to under-report, (2) solution sharing across semesters as an alternative explanation for behavioral changes, (3) Scala-specific findings may not generalize to other languages. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the Limitations section mentions Scala and one course, the paper does not explicitly state what the results do NOT show. The Conclusion (Section 8) extrapolates to workforce impacts ('junior developers') and broader educational implications without explicitly bounding the scope. The paper would benefit from a clear statement like 'these results apply only to this specific course/task/population.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw student submission data is not available for independent verification. No data repository or supplementary materials are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the data collection in detail: submissions from 10 semesters of the Cloud Computing course at CMU, Fall 2020 - Spring 2025, auto-graded PageRank task, students submit source code as often as they like before the deadline. The course structure, grading components, and team project setup are all described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not a recruitment-based study. The data comes from institutional records of all students enrolled in a university course. The population is defined by course enrollment, not active recruitment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the pipeline: 936 total enrollments → 721 who attempted PageRank task (filtering criterion: made at least one submission). Section 4.1 describes how source code files are concatenated and edit distances computed. The metrics derivation is documented in Sections 4.1 and 4.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This research is funded in part by the Carnegie Mellon-Accenture Center of Excellence in AI-Enabled Workforce Training (ACE-AI). The Cloud Computing course at CMU is sponsored in part by Amazon Web Service, Google Cloud Platform, and Microsoft Azure.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: three authors from Carnegie Mellon University and one from Udacity. The course being studied is at CMU where three of the four authors are affiliated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (Accenture via ACE-AI, and cloud providers AWS/Google/Microsoft) do not have a direct financial interest in whether LLMs are shown to help or harm student learning. The funding is for the center and course infrastructure, not contingent on particular findings about LLM effects."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. One author is affiliated with Udacity, an online education platform, which could have interests related to findings about AI and education, but no conflicts statement addresses this."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model on a benchmark. It is an observational study of student coding behavior."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model is being evaluated on a benchmark. This is an observational study of student submissions."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted. The study analyzes institutional data about student behavior."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is an analysis of existing institutional records (student code submissions), not a human subjects experiment. Students were not recruited or asked to participate in a study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "This is an analysis of existing course submission records. Students were not recruited as research subjects. However, the paper does not mention IRB approval or exemption status, which might be relevant given the analysis of student data."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is an analysis of institutional records, not a human subjects study. The paper does note that students are 'mostly graduate students' (Figure 3) from CMU's cloud computing course, which provides some population characterization."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study. The inclusion criterion for the analysis population (students who made at least one submission for the PageRank task) is documented in Section 3."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study, not an experiment with randomized conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational study analyzing historical data. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is an analysis of existing records, not a human subjects study with participants who could drop out of a study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "No AI inference is performed as part of this study's methodology. It is an observational analysis of student data."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "The study performs statistical analysis of student submissions. No significant compute budget is involved."
       }
     }
   },
   "claims": [
     {
       "claim": "Student coding behavior has changed significantly since Fall 2022: median Average Edit Distance increased by 300%, median Total Edit Distance increased by 500%, and median Number of Submissions increased by 50%.",
       "evidence": "Section 5.1 and Discussion Section 6, supported by Figures 4, 5, 6 showing distributions per semester. The increases are visually apparent in the box plots starting from s23.",
       "supported": "moderate"
     },
     {
       "claim": "There is a statistically significant negative correlation between Average Edit Distance and IP Score (corr=-0.16, p<0.001).",
       "evidence": "Section 5.3 and Figure 12. The correlation is computed across all enrollments with the specific magnitude quantified: 'for each additional 60 lines in Average Edit Distance, there is an associated 1% decrease in one's IP Score.'",
       "supported": "strong"
     },
     {
       "claim": "Average Submission Effect (score improvement per submission) has decreased steadily since s24, suggesting students make less effective changes between submissions.",
       "evidence": "Section 5.3, Figure 10 and Figure 11. Figure 11 shows an increasing proportion of submissions with no score change or decreased scores in recent semesters.",
       "supported": "moderate"
     },
     {
       "claim": "The negative correlation between Average Edit Distance and TP Score that existed Pre-ChatGPT disappeared Post-ChatGPT.",
       "evidence": "Section 5.3 and Figure 13. The pre-ChatGPT data shows a negative slope while post-ChatGPT data appears flat.",
       "supported": "moderate"
     },
     {
       "claim": "The behavioral changes are consistent with students using LLMs to generate code, though this cannot be definitively attributed to LLM usage.",
       "evidence": "Sections 5.1, 6, and 7. The paper bases this hypothesis on the observation that LLMs tend to introduce unnecessary edits when prompted to modify code. The Limitations section (Section 7) explicitly acknowledges they cannot confirm LLM usage.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["observational"],
   "key_findings": "This quasi-longitudinal study of 721 student enrollments across 10 semesters at CMU finds that coding behavior changed significantly after ChatGPT's release: edit distances increased dramatically (300-500% in medians), while score improvement per submission decreased. There is a statistically significant but small negative correlation (r=-0.16) between average edit distance and individual project scores. However, the authors cannot confirm whether students actually used LLMs, and task completion rates remained near-perfect, suggesting a ceiling effect that limits the sensitivity of performance metrics.",
   "red_flags": [
     {
       "flag": "Causal language without causal design",
       "detail": "Despite being an observational study with no ability to confirm LLM usage, the paper uses causal framing ('LLMs are impacting our students,' 'negatively affecting their learning outcomes,' 'changed how students interact'). The temporal correlation with ChatGPT's release is treated as suggestive of causation throughout, though alternative explanations (solution sharing, cohort differences, course evolution) are only briefly acknowledged."
     },
     {
       "flag": "Overgeneralization from narrow data",
       "detail": "The study examines one task (PageRank in Scala) in one course at one university, but the title, abstract, and conclusion generalize broadly to 'students,' 'junior developers,' and workforce productivity implications. The Conclusion speculates about industry effects ('increased requirements on companies' testing and quality assurance workflows') without evidence beyond this single classroom context."
     },
     {
       "flag": "Small effect size presented with alarm",
       "detail": "The negative correlation between Average Edit Distance and IP Score (r=-0.16) explains only about 2.6% of variance in IP scores. The paper contextualizes it as 'for each additional 60 lines in Average Edit Distance, there is an associated 1% decrease in one's IP Score,' which is very small, yet the abstract and conclusion frame findings as 'raising an alarm.'"
     },
     {
       "flag": "No data or code released",
       "detail": "Neither the student submission data nor the analysis code is released, making independent verification impossible. While student privacy is a valid concern, anonymized or aggregated data could have been provided."
     },
     {
       "flag": "Ceiling effect undermines core metric",
       "detail": "The paper acknowledges that nearly all students achieved perfect scores on the PageRank task and the median IP Score is 94%, creating a ceiling effect. This severely limits the ability to detect meaningful performance differences, yet behavioral conclusions about learning degradation are still drawn."
     }
   ],
   "cited_papers": [
     {
       "title": "The Influence of Artificial Intelligence Tools on Learning Outcomes in Computer Programming: A Systematic Review and Meta-Analysis",
       "authors": ["Manal Alanazi", "Ben Soh", "Halima Samra", "Alice Li"],
       "year": 2025,
       "doi": "10.3390/computers14050185",
       "relevance": "Meta-analysis of AI tools' impact on programming learning outcomes, directly relevant to understanding LLM effects on developer education."
     },
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "Studies the impact of AI tools on experienced developer productivity, finding that perceived productivity gains may not match actual gains."
     },
     {
       "title": "Beyond Code Generation: An Observational Study of ChatGPT Usage in Software Engineering Practice",
       "authors": ["Ranim Khojah", "Mazen Mohamad", "Philipp Leitner", "Francisco Gomes de Oliveira Neto"],
       "year": 2024,
       "doi": "10.1145/3660788",
       "relevance": "Observational study of ChatGPT usage patterns in professional software engineering practice."
     },
     {
       "title": "The impact of large language models on programming education and student learning outcomes",
       "authors": ["Gregor Jošt", "Viktor Taneski", "Sašo Karakatič"],
       "year": 2024,
       "relevance": "Studies the relationship between LLM reliance and final grades in programming education, finding a negative correlation."
     },
     {
       "title": "The Widening Gap: The Benefits and Harms of Generative AI for Novice Programmers",
       "authors": ["James Prather", "Brent Reeves", "Juho Leinonen", "Stephen MacNeil"],
       "year": 2024,
       "arxiv_id": "2405.17739",
       "relevance": "Examines how generative AI affects novice programmers, exploring both benefits and harms of over-reliance."
     },
     {
       "title": "An Empirical Study on Usage and Perceptions of LLMs in a Software Engineering Project",
       "authors": ["Sanka Rasnayaka", "Guanlin Wang", "Ridwan Shariffdeen", "Ganesh Neelakanta Iyer"],
       "year": 2024,
       "doi": "10.1145/3643795.3648379",
       "relevance": "Empirical study finding 40.5% of undergraduate teams used LLMs in software engineering projects."
     },
     {
       "title": "Would ChatGPT-facilitated programming mode impact college students' programming behaviors, performances, and perceptions? An empirical study",
       "authors": ["Dan Sun", "Azzeddine Boudouaia", "Chengcong Zhu", "Yan Li"],
       "year": 2024,
       "relevance": "Studies how ChatGPT affects student programming behaviors, performance, and perceptions including use patterns like 'reading feedback' and 'understanding code.'"
     },
     {
       "title": "Does ChatGPT Help With Introductory Programming? An Experiment of Students Using ChatGPT in CS1",
       "authors": ["Yuankai Xue", "Hanlin Chen", "Gina R. Bai", "Robert Tairas", "Yu Huang"],
       "year": 2024,
       "doi": "10.1145/3639474.3640076",
       "relevance": "Controlled experiment studying student ChatGPT usage vs. traditional resources in introductory programming."
     },
     {
       "title": "Codehelp: Using large language models with guardrails for scalable support in programming classes",
       "authors": ["Mark Liffiton", "Brad E Sheese", "Jaromir Savelka", "Paul Denny"],
       "year": 2023,
       "relevance": "Proposes guardrail-based LLM tutoring for programming education, relevant to understanding beneficial LLM usage."
     },
     {
       "title": "A Review on Vibe Coding: Fundamentals, State-of-the-art, Challenges and Future Directions",
       "authors": ["Partha Pratim Ray"],
       "year": 2025,
       "doi": "10.36227/techrxiv.174681482.27435614/v1",
       "relevance": "Reviews vibe coding practices where natural language directives orchestrate end-to-end software creation, relevant to understanding AI-assisted coding paradigms."
     }
   ]
 }