ai-research-survey

scan.json (24759B)

---

 {
   "paper": {
     "title": "Teaching Programming in the Age of Generative AI: Insights from Literature, Pedagogical Proposals, and Student Perspectives",
     "authors": [
       "Clemente Rubio-Manzano",
       "Jazna Meza",
       "Rodolfo Fernández-Santibáñez",
       "Christian Vidal-Castro"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2507.00108",
     "doi": "10.48550/arXiv.2507.00108"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code, tools, or repository links are provided anywhere in the paper. The proposed visual simulation approach is described conceptually but no implementation is released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Under 'Availability of data and material' the paper states 'Contact the corresponding author for data requests.' While Table I contains individual survey responses, no formal dataset is released. The data underlying the word cloud and emotion analysis is not provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses computational tools for generating the word cloud (Figure 11) and emotion analysis (Figure 12) but does not specify what software, libraries, or environment was used for these analyses."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided for either the survey methodology or the computational analyses (word cloud, emotion detection)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Survey results are reported as simple percentages (e.g., '86.1% indicated that code visualization was useful') with no confidence intervals or error bars, despite the small sample size (N=36) where uncertainty would be substantial."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are performed on any of the survey results. Claims about usefulness are based on raw percentages alone."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. The paper reports only raw percentages from the survey without any formal effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample size of 36 students is not justified. No power analysis is mentioned, and no acknowledgment is made that N=36 may be insufficient for drawing reliable conclusions."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a single cross-sectional survey, not a repeated experiment. There are no multiple runs or trials for which variance would be relevant."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No comparison is made against alternative teaching approaches, prior surveys of student perceptions, or a control group that did not receive the visual simulation intervention."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No experimental baselines are included, so the question of whether baselines are contemporary is inapplicable."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The pedagogical proposal is a single intervention (visual program simulation), not a multi-component system where components could be ablated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The primary evaluation metric is a single yes/no question about perceived usefulness. While grades and free-text comments are collected, they are not used as formal evaluation metrics for the intervention's effectiveness."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not produce system outputs that require human evaluation. The study itself IS a human survey, but this criterion asks about evaluating system outputs."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No test set is involved in this study. This is a survey-based pedagogical proposal, not a machine learning evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 10 provides a gender breakdown of students who found visual representation useful versus those who did not. Table I provides individual-level responses with grades and comments."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B discusses three specific areas where the approach could be improved based on negative feedback: need for standardization, importance of considering different learning styles, and the need to introduce the strategy earlier in the course. Individual negative comments are also reported (e.g., Student 34, Student 36)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 16.7% of students did not find the visualization useful, and includes specific negative comments such as Student 34's critique ('If you show this to someone who has never studied programming in their life, they won't understand') and Student 36's difficulty comparing visualizations with code."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are appropriately hedged: it promises to 'review the most relevant studies' (done in Section II), 'proposes enriching teaching and learning methodologies' (done in Sections III-IV), and presents 'preliminary context supporting the incorporation of visual simulations' (done in Section IV-B survey). No claims exceed what is presented."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implies visual simulation 'foster[s] a deeper understanding among students' (abstract) and that VPS 'promotes meaningful learning' (Section III-A). These are causal claims, but the study design—a single-group post-hoc survey with no control group, no pre-post measurement, and no randomization—is inadequate for causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title ('Teaching Programming in the Age of Generative AI') and abstract frame claims broadly for all programming education, but the empirical evidence comes from 36 students in one OOP course at one Chilean university. The paper even generalizes to 'Java (or other languages)' without evidence from other languages."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered for the survey results. Social desirability bias (students responding positively to their instructor's method), novelty effects, instructor quality, or selection bias from voluntary participation are not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures student self-reported perceptions of usefulness via a yes/no question but frames results as evidence that visual simulation is effective for actual understanding. The distinction between 'students say it helped' and 'it actually improved comprehension' is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No AI models are used in the paper's own experiments. The paper discusses LLMs conceptually but does not run any model-based experiments."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in this study. The paper discusses LLM-based code generation conceptually but does not use prompting in its own methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameters are involved. This is a pedagogical proposal and survey study, not a computational experiment."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how student comments were preprocessed for the word cloud (Figure 11) or what tool/method was used for the emotion analysis (Figure 12). The transformation from raw comments to these visualizations is undocumented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper moves directly from survey results to 'Conclusions and Future Works' (Section V) without discussing limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. Key threats such as the tiny sample size, lack of control group, potential instructor bias, voluntary response bias, and single-institution scope are not acknowledged."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what the results do NOT show or what populations/settings are excluded from its claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Table I provides all 36 individual survey responses including yes/no answers, grades, and free-text comments. This allows independent verification of the reported results (and reveals inconsistencies in the reported percentages)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the three survey questions (Section IV-B) and states 36 students responded, but does not describe when the survey was administered, how (online/paper), whether it was voluntary, during or after the course, or any other details of the data collection procedure."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "No recruitment methods are described. The paper says students 'who took the object-oriented programming course' but does not explain how they were recruited, whether participation was voluntary, or what proportion of the class responded."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of the data pipeline from raw survey responses to the word cloud (Figure 11) or emotion analysis (Figure 12). The tools, methods, and parameters used for these analyses are not described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Funding section explicitly states: 'This study was not supported by a grant.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Universidad del Bío-Bío and DuocUC, both in Concepción, Chile."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "The study is unfunded, so the question of funder independence is not applicable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "Under Competing Interests, the paper states: 'Authors declare that they have no competing interests.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model's capability on a benchmark. It is a pedagogical proposal with a student survey, so contamination concerns are not applicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation is performed, so train/test overlap is not applicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed, so benchmark contamination is not applicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned. The study surveyed 36 students but there is no reference to any pre-registration platform."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "Under 'Ethics approval and consent to participate' the paper states 'Not applicable.' This is concerning since data was collected from students (a potentially vulnerable population), yet no ethics review was sought or mentioned."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only gender is shown (Figure 10 pie chart). Key demographics for a CS education study—year of study, age, prior programming experience—are not reported. Participants are identified only as OOP students at a Chilean university."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria are stated. The paper says only that respondents were students 'who took the object-oriented programming course' with no further selection criteria or screening described."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a cross-sectional survey, not an experimental study with treatment/control groups, so randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a cross-sectional survey, not an experimental study, so blinding is not applicable."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports 36 respondents but does not state how many students were enrolled in the course. The response rate is unknown, making it impossible to assess selection/attrition bias."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a pedagogical proposal and survey paper, not a system with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a pedagogical proposal and survey paper with no significant compute requirements."
       }
     }
   },
   "scan_version": 3,
   "active_modules": [],
   "claims": [
     {
       "claim": "86.1% of students indicated that code visualization was useful for understanding object and reference concepts in the OOP course.",
       "evidence": "Section IV-B survey of 36 students, Figure 10, Table I. However, counting Table I directly yields 32/36 = 88.9% YES and 4/36 = 11.1% NO, which contradicts the reported 86.1% and 16.7% (which also do not sum to 100%).",
       "supported": "weak"
     },
     {
       "claim": "LLMs lack true understanding of the code they generate, operating on statistical correlations rather than semantic understanding.",
       "evidence": "Section I cites Shojaee et al. 2025 [4] ('The illusion of thinking') and Valmeekam et al. 2022 [5] ('Large language models still can't plan') to support this claim. No original evidence is presented.",
       "supported": "moderate"
     },
     {
       "claim": "Visual program simulation (VPS) is an effective pedagogical tool for teaching programming that promotes meaningful learning.",
       "evidence": "Section III reviews prior literature on visualization in education (Mayer, Python Tutor, Marton et al.) and Section IV-B presents a preliminary survey. The survey has no control group, no pre-post measurement, and N=36.",
       "supported": "weak"
     },
     {
       "claim": "Assessment in programming courses should shift from evaluating code functionality to evaluating student understanding of code execution.",
       "evidence": "Sections III-IV present this as a pedagogical proposal supported by literature on visualization and the argument that LLMs cannot truly understand code. No comparative evidence is provided showing this approach yields better learning outcomes.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "qualitative",
     "theoretical"
   ],
   "key_findings": "This paper reviews advantages and disadvantages of LLMs in programming education and proposes visual program simulation (VPS) as a complementary assessment strategy focusing on code comprehension over code writing. A preliminary survey of 36 OOP students found that the majority reported visual representations were useful for understanding object and reference concepts, though the reported percentages (86.1%) do not match the raw data in Table I (88.9%). The paper argues that since LLMs lack true code understanding, assessment should verify students' ability to mentally simulate program execution using box-and-arrow visual representations.",
   "red_flags": [
     {
       "flag": "Data inconsistency",
       "detail": "The paper reports 86.1% of students found visualization useful and 16.7% did not (summing to 102.8%), but counting Table I yields 32 YES and 4 NO out of 36 (88.9% and 11.1%). The reported percentages are internally inconsistent and do not match the published data."
     },
     {
       "flag": "Tiny uncontrolled sample",
       "detail": "N=36 students with no control group, no pre-post measurement, no randomization, and a single yes/no question as the primary metric. This design cannot support claims about pedagogical effectiveness."
     },
     {
       "flag": "Instructor bias risk",
       "detail": "Students were surveyed about the usefulness of their own instructor's teaching method. Social desirability bias is highly likely, especially given no anonymity safeguards are described beyond the acknowledgment noting anonymous participation."
     },
     {
       "flag": "Unknown response rate",
       "detail": "The paper does not report how many students were enrolled in the course versus how many responded (36). If the response rate is low, the results may suffer from severe self-selection bias."
     },
     {
       "flag": "No ethics review",
       "detail": "The paper explicitly states 'Not applicable' for ethics approval despite collecting data from students, a potentially vulnerable population whose grades could be influenced by the instructor-researchers."
     },
     {
       "flag": "Narrative literature review without systematic methodology",
       "detail": "Section II reviews only 5-6 studies on LLMs in programming education with no structured search protocol, inclusion/exclusion criteria, or quality assessment of sources. The review is selective and may not represent the full body of evidence."
     },
     {
       "flag": "Claims outrun evidence",
       "detail": "Broad claims about programming education ('Teaching Programming in the Age of Generative AI') are supported only by a tiny survey at one Chilean university in one Java OOP course. The paper generalizes to 'other languages' without evidence."
     },
     {
       "flag": "Sloppy reference",
       "detail": "Reference [3] lists the venue as 'Name of Journal' — an apparent placeholder that was never corrected, suggesting incomplete quality control in manuscript preparation."
     }
   ],
   "cited_papers": [
     {
       "title": "Programming education and learner motivation in the age of generative AI: student and educator perspectives",
       "authors": ["S. Boguslawski", "R. Deer", "M. G. Dawson"],
       "year": 2025,
       "relevance": "Directly studies the relationship between student motivation and LLM use in introductory programming courses."
     },
     {
       "title": "Generative AI in introductory programming",
       "authors": ["B. A. Becker", "M. Craig", "P. Denny", "H. Keuning", "N. Kiesler", "J. Leinonen", "A. Luxton-Reilly", "L. Malmi", "J. Prather", "K. Quille"],
       "year": 2023,
       "relevance": "Argues for the transformative potential of generative AI in introductory programming education with implications for CS curricula."
     },
     {
       "title": "The illusion of thinking: Understanding the strengths and limitations of reasoning models via the lens of problem complexity",
       "authors": ["P. Shojaee", "I. Mirzadeh", "K. Alizadeh", "M. Horton", "S. Bengio", "M. Farajtabar"],
       "year": 2025,
       "arxiv_id": "2506.06941",
       "relevance": "Evaluates reasoning limitations of LLMs, directly relevant to claims about AI coding capability boundaries."
     },
     {
       "title": "Large language models still can't plan (a benchmark for LLMs on planning and reasoning about change)",
       "authors": ["K. Valmeekam", "A. Olmo", "S. Sreedharan", "S. Kambhampati"],
       "year": 2022,
       "relevance": "Benchmarks LLM planning and reasoning limitations, relevant to the argument that LLMs lack true code understanding."
     },
     {
       "title": "Teaching and learning computer programming using ChatGPT: A rapid review of literature amid the rise of generative AI technologies",
       "authors": ["M. B. Garcia"],
       "year": 2025,
       "relevance": "Reviews literature on using ChatGPT for teaching programming, directly within the survey scope of LLM impact on programming education."
     },
     {
       "title": "The impact of AI use in programming courses on critical thinking skills",
       "authors": ["C. J. S. F. Clarke", "A. Konak"],
       "year": 2025,
       "relevance": "Studies the effect of AI tools on critical thinking in programming courses, relevant to concerns about LLM impact on learning."
     },
     {
       "title": "Online Python Tutor: embeddable web-based program visualization for CS education",
       "authors": ["P. J. Guo"],
       "year": 2013,
       "relevance": "Foundational work on code visualization tools for programming education, directly relevant to the visual simulation approach discussed."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The VPS pedagogical concept could interest CS educators, but no tool, software, or materials are released for practical use."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "The proposal that visualization aids learning is well-established; nothing here challenges conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI risk or security concerns are raised; the paper discusses pedagogical methodology."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims; the paper takes a measured pedagogical position."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is provided; the visual simulations are only hand-drawn figures in the paper."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors are from Universidad del Bío-Bío and DuocUC in Chile, not widely known in the AI research community."
     }
   }
 }