ai-research-survey

scan.json (40509B)

---

 {
   "paper": {
     "title": "The Robots are Here: Navigating the Generative AI Revolution in Computing Education",
     "authors": [
       "James Prather",
       "Paul Denny",
       "Juho Leinonen",
       "Brett A. Becker",
       "Ibrahim Albluwi",
       "Michelle Craig",
       "Hieke Keuning",
       "Natalie Kiesler",
       "Tobias Kohn",
       "Andrew Luxton-Reilly",
       "Stephen MacNeil",
       "Andrew Petersen",
       "Raymond Pettit",
       "Brent N. Reeves",
       "Jaromir Savelka"
     ],
     "year": 2023,
     "venue": "ITiCSE 2023",
     "arxiv_id": "2310.00658",
     "doi": "10.1145/3587103.3594206"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage", "survey_methodology"],
   "methodology_tags": ["meta-analysis", "qualitative", "observational", "benchmark-eval"],
   "key_findings": "This large working group report synthesizes 71 papers on LLMs in computing education and finds that LLMs perform at or above average student level on introductory programming tasks, with GPT-4 dramatically outperforming Codex within two years. A survey of 228 respondents across 20 countries shows students and instructors agree GenAI use should have some restrictions, but disagree on the clarity of existing policies. Interviews with 22 educators reveal a shift toward process-over-product assessment, greater emphasis on code reading, and reduced weight on unsupervised homework. Available computing education benchmarks are heavily biased toward introductory Python problems, limiting the scope of current LLM evaluations.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No analysis code, scripts, or tools are released. The paper releases benchmark problem descriptions and test cases on OSF (Section 8.3, footnote 5) but no code for the survey analysis, interview analysis, or benchmarking pipeline."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper releases the problem descriptions and test cases from the Finnie-Ansley et al. replication on OSF (https://osf.io/bu9h3/). The benchmarking also uses publicly available datasets (HumanEval, APPS, FalconCode). However, survey data and interview transcripts are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency information is provided for reproducing the benchmarking experiments or survey/interview analyses."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The benchmarking methodology is described in prose (Sections 8.3-8.4) but lacks concrete commands or scripts to reproduce results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for any results. Survey responses are presented as percentages and Likert distributions. Benchmarking results are raw pass/fail counts. APPS results (Table 10) report averages without uncertainty bounds."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used despite making comparative claims. Student vs. instructor survey responses are compared descriptively without formal tests. LLM performance comparisons across models (Table 9, Table 10) use only raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes (Cohen's d, odds ratios, etc.) are reported. The benchmarking comparisons report raw percentages (e.g., GPT-4 at 51.5% vs GPT-3.5 at 39.2% on APPS) but without formal effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes. The survey had 57 instructors and 171 students with no power analysis or sample size rationale. The interview sample of 22 educators is described but not justified. The paper acknowledges selection bias (Section 4.2.1) but does not discuss statistical adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported. Benchmarking uses single-run results (temperature 0.9 with up to 10 attempts, temperature 0.0 for APPS). No spread measures across experimental runs are provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The replication study (Section 8.3) explicitly compares GPT-4, GPT-3.5-turbo, and Copilot against the original Codex results from Finnie-Ansley et al. Figure 3 directly compares original vs replication. The literature review compares quality metrics against Hellas et al. (Table 5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The replication uses GPT-4 (state-of-the-art at the time), GPT-3.5-turbo (free ChatGPT model), and GitHub Copilot, all contemporary models. The comparison against Codex is the explicit purpose of the replication to show improvement over time."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a multi-method working group report (literature review, survey, interviews, benchmarking), not a single system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "For APPS (Table 10), both 'test case average' and 'strict accuracy' are reported. For FalconCode (Table 11), 'Full' and 'Clean' success rates are reported. The replication reports both attempt number and pass/fail. The literature review uses four quality metrics (Table 5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The benchmarking evaluation is entirely automated via test cases. While the authors manually analyzed failure cases (e.g., FalconCode thematic analysis of failures, Table 12), this is error analysis rather than human evaluation of LLM outputs. The schema specifies that automated pass/fail on test suites is NO."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The models are evaluated on established benchmark problems without any tuning. The replication uses exam problems from Finnie-Ansley et al. APPS and FalconCode are standard benchmarks. No model tuning or development decisions were made on these test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 9 breaks down replication results per problem. Table 10 breaks APPS results by difficulty level (introductory/interview/competition). Table 11 breaks FalconCode by assignment type (Skill/Lab/Project). Table 8 categorizes benchmark problems by course level."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.3 discusses specific problems models failed on (T2-Q12 bar graph problem, Rainfall variants) with analysis of why. Section 8.4.2 contains detailed thematic analysis of GPT-4 failures on FalconCode (Table 12), identifying missing instructions, incorrect structure, and genuinely incorrect solutions."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results reported: all models failed on T2-Q12; Copilot failed on 3 exam problems and 3 rainfall variants; GPT-4 only achieved 45.4% on FalconCode raw (Table 11); Section 8.4.2 documents extensive dataset quality issues. The APPS competition-level problems showed poor performance (28.7% for GPT-4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract lists five specific contributions (literature review of 71 papers, survey across 20 countries, 22 educator interviews, ethics analysis via ACM Code of Ethics, LLM benchmarking). All five are delivered in corresponding sections (3, 4, 5, 6-7, 8). Claims are descriptive and matched by the paper's content."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims of its own. It reports descriptive survey findings, qualitative interview themes, literature synthesis, and benchmarking comparisons. Performance comparisons between models ('GPT-4 outperforms Codex') are direct measurement comparisons, not causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Each component includes explicit scope limitations. Section 3.7 bounds the literature review to April-August 2023. Section 4.2.1 acknowledges selection bias in the survey sample. Section 5.8 notes geographic skew (55% US). Section 8.5.1 acknowledges the benchmarking is limited to code generation tasks and mostly introductory Python."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "For benchmarking, data contamination is discussed as an alternative explanation for high performance: 'There is a large chance that solutions for it have been included in training recent models' (Section 8.4.1). For the survey, selection bias toward engaged educators and high-achieving students is discussed (Section 4.2.1). For FalconCode, missing information rather than model failure explains many poor results (Section 8.4.2)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.5.1 explicitly distinguishes between what test cases measure and what educators care about: 'testing generated code can be easily automated by running test cases, although this might not capture all aspects relevant to computing educators, such as code quality and suitability of the solution with regards to which concepts the student has learned so far.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4', 'GPT-3.5-turbo', and 'GitHub Copilot' without specifying exact versions or snapshot dates (e.g., no 'gpt-4-0613'). Section 8.3 notes 'GPT-4' and 'GPT-3.5-turbo' without version identifiers. The original study used 'code-davinci-001' which is more specific."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.4.1 provides the full system prompt used for APPS and FalconCode experiments, including the example input/output format. Section 8.3 describes the prompt format: 'problem description surrounded by triple-quotes as was done in the original study.' The prompt text is fully reproducible."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.3 states 'temperature value of 0.9 similar to the original study.' Section 8.4.1 reports 'temperature=0.0, max_tokens=4000, and default values for Top P (1), Frequency penalty (0), and Presence penalty (0).' Key generation parameters are fully specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The benchmarking sends direct prompts to models and collects completions. Copilot is used via IDE plugin as a black box."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The literature review pipeline is documented with counts at each stage (Figure 1): 19 seed papers → 513 snowballed → inclusion criteria applied → 46 included → 8 excluded after deep review → 38 → second snowballing → 71 final papers. For FalconCode, de-duplication steps are documented (661 → 422 by ID → 385 by prompt). Filtering criteria are stated at each stage (Section 3.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Multiple dedicated limitations subsections are present: Section 3.7 (literature review limitations and threats to validity), Section 5.8 (interview limitations and threats to validity), and Section 8.5.1 (benchmarking limitations). Each contains substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: literature review was bounded to April-August 2023 and used only one step of snowballing (Section 3.7); interview sample was geographically skewed (55% US) with only 14% women (Section 5.8); Codex was deprecated during the study (Table 4); FalconCode dataset had missing starter code and data files causing artificial failures (Section 8.4.2)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While limitations are discussed, the paper does not explicitly state what claims it is NOT making. The title ('Navigating the Generative AI Revolution in Computing Education') is broad, and the paper does not provide a structured statement of what the results do not show. Limitations are framed as caveats rather than explicit scope exclusions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Survey responses are not released. Interview transcripts were deleted per ethics protocol (Section 5.1: 'we deleted all audio and video recordings'). Only the benchmarking problem descriptions and test cases are released on OSF. The primary datasets (survey n=228, interview transcripts) are not available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Survey design and distribution described in Section 4.2 (SIGCSE mailing lists, snowball sampling). Interview methodology described in Section 5.1 (semi-structured, Zoom, transcription process). Literature review search described in Section 3.1 (databases, keywords, dates). Benchmarking data collection described in Sections 8.3-8.4."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Survey recruitment via SIGCSE mailing lists for instructors (n=57) and snowball sampling for students (n=171) is described in Section 4.2.1, including an attempt to mitigate response bias by including language targeting struggling students. Interview recruitment via purposeful sampling through author networks, SIGCSE mailing list, and survey volunteers is described in Section 5.1."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The literature review pipeline is fully documented in Figure 1 with counts at each stage and criteria for inclusion/exclusion. Survey analysis process (thematic analysis) is described in Section 4.2.3. Interview analysis pipeline (verbatim transcription → thematic analysis → theme review) is described in Section 5.1. FalconCode pipeline (extraction → de-duplication → generation → testing → analysis) is documented in Section 8.4.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The acknowledgments section thanks interviewees but does not mention any grants, corporate sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All 15 author affiliations are clearly listed on the first page, spanning universities in the USA, New Zealand, Ireland, Jordan, Canada, Netherlands, Germany, Austria."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funder is disclosed, so independence cannot be assessed. The work appears to be unfunded academic research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for GPT-4, GPT-3.5-turbo, or GitHub Copilot. The paper evaluates these models on benchmarks without specifying when their training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "For APPS, Section 8.4.1 discusses overlap: 'A major downside of the APPS dataset is that it is a public dataset with problems from popular online coding websites. There is a large chance that solutions for it have been included in training recent models.' The replication explicitly releases new data to facilitate future uncontaminated testing."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The APPS contamination risk is explicitly acknowledged (Section 8.4.1). For the replication, the authors note the dataset was not previously public and release it specifically to support future replication (Section 8.3). However, no formal contamination analysis (canary strings, membership inference) is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration for the survey or interview study. No link to OSF, AsPredicted, or any pre-registration platform."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states the interview protocol was 'submitted to the University of Toronto Research Ethics Board, who approved it prior to the study.' Audio and video recordings were deleted in accordance with this protocol."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3.1 reports detailed demographics: instructors from 12 countries, avg 18.2 years experience, 77.2% men, class sizes; students from 17 countries, year of study (48.5% first year), avg 4.6 courses, majors (42.7% CS). Table 6 shows interviewee countries."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "For interviews, Section 5.1 states: 'The most important criterion for inclusion was that educators would have concrete plans or views toward changing their current course structure, assessment, or classroom practices in light of LLMs.' Survey recruitment targeted computing education mailing lists (Section 4.2.1)."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study. The survey and interviews are observational/qualitative studies without treatment conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable for cross-sectional surveys and qualitative interviews. No treatment conditions exist that would require blinding."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition or dropout information is reported. The paper states final counts (57 instructors, 171 students, 22 interviewees) but does not report how many were initially contacted, started the survey, or declined interviews."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or latency figures are reported for the benchmarking experiments despite running GPT-4, GPT-3.5-turbo, and Copilot across hundreds of problems with up to 10 attempts each."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. No information on API spend, GPU hours, or total wall-clock time for the benchmarking experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Single-run results are reported. For GPT-4/GPT-3.5, temperature 0.9 is used with up to 10 attempts, but no sensitivity analysis across different seeds or runs is performed. For APPS, temperature 0.0 produces deterministic output from a single run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.3 states 'We generate up to ten solutions for each problem, stopping if the LLM creates a solution that passes the tests.' Section 8.4.1 implicitly states one run per problem at temperature=0.0. The attempt protocol is clearly specified."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search was conducted. The replication uses temperature 0.9 from the original study, and APPS uses temperature 0.0, but no exploration of alternative settings is reported or justified beyond following the original study's choices."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "For the replication (Section 8.3), using temperature 0.9 is justified as matching 'the original study.' For APPS (Section 8.4.1), temperature=0.0 and max_tokens=4000 are stated with defaults. While the APPS temperature choice isn't explicitly justified, the replication's rationale is clear."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so no multiple comparison correction is needed."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The authors are not evaluating their own system. They evaluate third-party models (GPT-4, GPT-3.5, Copilot) on existing benchmarks. Self-comparison bias does not apply."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4, GPT-3.5, and Copilot have vastly different computational costs, but performance is not reported as a function of compute. The models are compared on accuracy alone without discussing cost-performance tradeoffs."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.2 provides extensive analysis of what the benchmarks actually measure, finding most problems cover only introductory Python (Table 8: 98.8% of HumanEval, 100% of FalconCode are introductory). Section 8.5 discusses that available datasets 'may not reflect the kinds of problems student programmers solve in upper year courses.'"
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4 and GPT-3.5 are tested via API with identical prompting, but Copilot is tested via a different interface (VS Code plugin with IDE completion). The paper describes the different interfaces (Section 8.3) but does not explicitly discuss how this scaffold difference could confound the model comparison."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.4.1 explicitly addresses temporal leakage for APPS: 'A major downside of the APPS dataset is that it is a public dataset with problems from popular online coding websites. There is a large chance that solutions for it have been included in training recent models.'"
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks answer information through context (e.g., whether problem descriptions contain hints that wouldn't be available in authentic educational settings)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark problems share structural similarities, come from the same sources, or have near-duplicate variants that could inflate performance estimates."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is used. The contamination concern for APPS is discussed conceptually but no canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines are applied."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The literature review follows a structured scoping review protocol with defined search queries across three databases (ACM, Taylor & Francis, IEEE), explicit date filters (January 2021), inclusion/exclusion criteria (5 criteria listed in Section 3.1.2), two rounds of snowballing, dual evaluation with third-evaluator resolution, and a documented pipeline (Figure 1)."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 assesses quality metrics of included papers using criteria adapted from Hellas et al.: clearly defined research question (Yes:44, No:18), research process described (Yes:55), results detail (Yes:57), threats to validity (Yes in separate section:38, Yes but not separate:15, No:18)."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. While the paper acknowledges including arXiv and grey literature 'driven by pragmatism' (Section 3.7), it does not discuss whether published papers in the review skew toward positive results about LLMs, nor does it use funnel plots or other bias detection methods."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs perform at a level equivalent to or better than average students on introductory code generation tasks.",
       "evidence": "Section 3.3.1 synthesizes findings from 35+ papers evaluating LLM performance, and the replication (Section 8.3, Figure 3) shows GPT-4 solving nearly all CS1 exam problems, performing as 'one of the top students in the class.'",
       "supported": "strong"
     },
     {
       "claim": "GPT-4 dramatically outperforms Codex on CS1 exam problems within two years.",
       "evidence": "Section 8.3 and Table 9: GPT-4 solved all problems except T2-Q12, mostly on the first attempt. The original Codex study showed performance 'similar to students in the top quartile' while GPT-4 'would have been one of the top students in the class.' Figure 3 visually compares original vs replication.",
       "supported": "strong"
     },
     {
       "claim": "Students and instructors are generally aligned in their belief that some restrictions should be placed on GenAI tool use in coursework.",
       "evidence": "Section 4.3.2 and Figure 2: Both groups show similar patterns on Likert responses about restrictions. However, they diverge on policy clarity — instructors find policies less clear than students.",
       "supported": "moderate"
     },
     {
       "claim": "Around half of instructors (50.8%) believe many or almost all of their students are using GenAI tools unethically.",
       "evidence": "Section 4.3.4 reports this figure directly from the instructor survey (n=57). The question asked 'To what extent do you think students at your school are using GenAI tools in ways that you would not approve of?'",
       "supported": "moderate"
     },
     {
       "claim": "Available computing education benchmarks are heavily biased toward introductory Python problems.",
       "evidence": "Section 8.2, Table 8: 98.8% of HumanEval, 100% of FalconCode, and 83.3% of the CS2 dataset problems are classified as introductory level. Table 7 shows most datasets focus on Python. Manual tagging by experienced instructors confirmed the homogeneity.",
       "supported": "strong"
     },
     {
       "claim": "The most common concern about LLMs in computing education is that students will become over-reliant on them.",
       "evidence": "Section 3.6.1: 'The most common concern expressed by authors about student learning was the potential for students to become over-reliant on generative AI tools to solve problems [34, 82, 115, 118, 158] and assist in debugging code [116, 140, 187].' Corroborated by survey (Section 4.3) and interviews (Section 5).",
       "supported": "strong"
     },
     {
       "claim": "AI text detection tools are unreliable and produce many false positives.",
       "evidence": "Section 3.3.1 cites Orenstrakh et al. [134]: 'tools that assess whether a given text was generated by an LLM show a large number of false positives and should not be trusted blindly.' Noted to be 'even worse when evaluating code.'",
       "supported": "moderate"
     },
     {
       "claim": "Educators are shifting toward process-over-product assessment and increasing the weight of invigilated exams.",
       "evidence": "Section 5.4: Multiple interviewees describe reducing homework weight (one changed from 50% to 0%), adding coding interviews, oral exams (Jean Mehta), and reflection assignments. Section 5.4.3 describes the shift to evaluating learning processes. Based on 22 interviews.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Selection bias in survey sample",
       "detail": "Instructors were recruited via SIGCSE mailing lists, likely overrepresenting educators highly engaged with computing education research. Students were recruited via snowball sampling through those instructors. Section 4.2.1 acknowledges this: 'the resulting sample likely results in a selection bias of instructors who are particularly invested in computing education.' The sample of 57 instructors and 171 students across 20 countries may not represent the global computing education community."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "Student vs. instructor survey responses are compared descriptively (Section 4.3.2) with language like 'some important differences emerged' but without statistical tests. The paper cannot distinguish real differences from sampling noise. Similarly, benchmarking comparisons between models lack significance tests."
     },
     {
       "flag": "Interview sample geographic skew",
       "detail": "22 educators, 55% from USA, only 1 each from Asia, Oceania, and South America, none from Africa (Section 5.8, Table 6). Only 14% women. Findings about how 'educators are changing' are heavily weighted toward US perspectives."
     },
     {
       "flag": "Benchmark contamination unaddressed for replication",
       "detail": "While APPS contamination is discussed, the CS1 exam problems used in the Finnie-Ansley replication may have entered GPT-4's training data through academic dissemination, citations, or the original paper's availability. The high GPT-4 performance could be partly explained by memorization rather than genuine problem-solving capability."
     },
     {
       "flag": "Literature review explicitly not systematic",
       "detail": "Section 3.1 states 'We explicitly considered but decided not to perform a systematic review.' While a scoping review is a valid choice, the one-step snowballing from only 10 seed papers, combined with the acknowledged rapid pace of publication, means important work may be systematically missed."
     }
   ],
   "cited_papers": [
     {
       "title": "The Robots Are Coming: Exploring the Implications of OpenAI Codex on Introductory Programming",
       "authors": ["James Finnie-Ansley", "Paul Denny", "Brett A. Becker", "Andrew Luxton-Reilly", "James Prather"],
       "year": 2022,
       "doi": "10.1145/3511861.3511863",
       "relevance": "Seminal paper evaluating Codex on CS1 programming exercises, replicated in this working group report with newer models."
     },
     {
       "title": "Automatic Generation of Programming Exercises and Code Explanations Using Large Language Models",
       "authors": ["Sami Sarsa", "Paul Denny", "Arto Hellas", "Juho Leinonen"],
       "year": 2022,
       "doi": "10.1145/3501385.3543957",
       "relevance": "Early evaluation of LLMs for generating educational content (programming exercises and code explanations)."
     },
     {
       "title": "Studying the Effect of AI Code Generators on Supporting Novice Learners in Introductory Programming",
       "authors": ["Majeed Kazemitabaar", "Justin Chow", "Carl Ka To Ma", "Barbara J. Ericson", "David Weintrop", "Tovi Grossman"],
       "year": 2023,
       "doi": "10.1145/3544548.3580919",
       "relevance": "Human study finding AI code generators improve novice productivity (1.15x progress, 0.57x time) without negative learning effects."
     },
     {
       "title": "Repairing Bugs in Python Assignments Using Large Language Models",
       "authors": ["Jialu Zhang", "José Cambronero", "Sumit Gulwani", "Vu Le", "Ruzica Piskac", "Gustavo Soares", "Gust Verbruggen"],
       "year": 2022,
       "arxiv_id": "2209.14876",
       "relevance": "Demonstrated 96.5% repair rate using Codex with few-shot examples and iterative prompting for student bug repair."
     },
     {
       "title": "Using Large Language Models to Enhance Programming Error Messages",
       "authors": ["Juho Leinonen", "Arto Hellas", "Sami Sarsa", "Brent Reeves", "Paul Denny", "James Prather", "Brett A. Becker"],
       "year": 2023,
       "doi": "10.1145/3545945.3569770",
       "relevance": "Evaluated Codex for enhancing programming error messages, finding success in 54% of cases."
     },
     {
       "title": "Lost at C: A User Study on the Security Implications of Large Language Model Code Assistants",
       "authors": ["Gustavo Sandoval", "Hammond Pearce", "Teo Nys", "Ramesh Karri", "Siddharth Garg", "Brendan Dolan-Gavitt"],
       "year": 2023,
       "relevance": "User study on security implications of LLM code assistants, finding no significant new security risks introduced."
     },
     {
       "title": "Detecting LLM-Generated Text in Computing Education: A Comparative Study for ChatGPT Cases",
       "authors": ["Michael Sheinman Orenstrakh", "Oscar Karnalim", "Carlos Anibal Suarez", "Michael Liut"],
       "year": 2023,
       "arxiv_id": "2307.07411",
       "relevance": "Evaluated AI text detection tools in computing education, finding high false positive rates especially for code."
     },
     {
       "title": "Generative AI for Programming Education: Benchmarking ChatGPT, GPT-4, and Human Tutors",
       "authors": ["Tung Phung", "Victor-Alexandru Pădurean", "José Cambronero", "Sumit Gulwani", "Tobias Kohn", "Rupak Majumdar", "Adish Singla", "Gustavo Soares"],
       "year": 2023,
       "relevance": "Benchmarked GPT-4 against human tutors for programming education, finding performance approaching human tutors."
     },
     {
       "title": "Practical and Ethical Challenges of Large Language Models in Education: A Systematic Literature Review",
       "authors": ["Lixiang Yan", "Lele Sha", "Linxuan Zhao", "Yuheng Li", "Roberto Martinez-Maldonado", "Guanliang Chen", "Xinyu Li", "Yueqiao Jin", "Dragan Gašević"],
       "year": 2023,
       "arxiv_id": "2303.13379",
       "relevance": "Systematic review of LLM challenges in education covering assessment, grading, and ethical concerns."
     },
     {
       "title": "Computing Education in the Era of Generative AI",
       "authors": ["Paul Denny", "James Prather", "Brett A. Becker", "James Finnie-Ansley", "Arto Hellas", "Juho Leinonen", "Andrew Luxton-Reilly", "Brent N. Reeves", "Eddie Antonio Santos", "Sami Sarsa"],
       "year": 2023,
       "arxiv_id": "2306.02608",
       "relevance": "Companion survey paper on how computing education should adapt to generative AI."
     },
     {
       "title": "Grounded Copilot: How Programmers Interact with Code-Generating Models",
       "authors": ["Shraddha Barke", "Michael B. James", "Nadia Polikarpova"],
       "year": 2023,
       "relevance": "Identified programmer interaction patterns with code generators (exploration and acceleration modes)."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced Codex and the HumanEval benchmark, foundational for LLM code generation evaluation."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Demonstrated that existing benchmarks have insufficient test cases, allowing incorrect LLM solutions to pass — introduced EvalPlus."
     },
     {
       "title": "Programming Is Hard - Or at Least It Used to Be: Educational Opportunities and Challenges of AI Code Generation",
       "authors": ["Brett Becker", "James Prather", "Paul Denny", "Andrew Luxton-Reilly", "James Finnie-Ansley", "Eddie Antonio Santos"],
       "year": 2023,
       "relevance": "Position paper on educational opportunities and challenges of AI code generation in computing education."
     },
     {
       "title": "Measuring Coding Challenge Competence With APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "relevance": "Introduced the APPS benchmark dataset (10,000 programming problems) used in this paper's benchmarking analysis."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Immediately useful for computing educators with concrete advice sections for educators, students, and policy makers, plus a sample student handout (Appendix D)."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Generally confirms expected trends — LLMs are improving rapidly, educators are concerned about cheating, students are using the tools — rather than challenging conventional wisdom."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Discusses academic integrity concerns and student over-reliance on AI tools, but does not raise novel AI risk or security concerns."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Some tension between 'ban' and 'embrace' perspectives from educators, but presented as balanced analysis rather than controversy."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, tool, or demo to try. The released OSF data is benchmark problems and test cases, not an interactive artifact."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Evaluates GPT-4, ChatGPT, and GitHub Copilot (high brand recognition). Published at ITiCSE, a major computing education venue. Authors are well-known in computing education research."
     }
   }
 }