ai-research-survey

scan.json (25908B)

---

 {
   "paper": {
     "title": "Beyond the Hype: A Comprehensive Review of Current Trends in Generative AI Research, Teaching Practices, and Tools",
     "authors": [
       "James Prather",
       "Juho Leinonen",
       "Natalie Kiesler",
       "Jamie Gorson Benario",
       "Sam Lau",
       "Stephen MacNeil",
       "Narges Norouzi",
       "Simone Opel",
       "Vee Pettit",
       "Leo Porter",
       "Brent N. Reeves",
       "Jaromir Savelka",
       "David H. Smith IV",
       "Sven Strickroth",
       "Daniel Zingaro"
     ],
     "year": 2024,
     "venue": "ITiCSE-WGR 2024",
     "arxiv_id": "2412.14732",
     "doi": "10.1145/XXXXXXX"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or analysis scripts are mentioned as released. The paper does not provide a repository URL for any code used in data analysis."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The systematic literature review data is published on OSF: https://osf.io/wpxjb/?view_only=cebf366db7f5423792b39de754972400 (stated in the Data Availability section)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. The paper does not describe any computational environment setup for its analyses."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While the SLR methodology is described, there are no scripts or instructions for reproducing the quantitative analyses."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports only descriptive statistics (percentages, medians, IQRs for some distributions). No confidence intervals or error bars are reported for the main survey results or comparisons."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes several comparative claims (e.g., educators underestimate developer GenAI usage, tools with guardrails produce more positive results) but uses no statistical significance tests to support these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. Comparisons are made using raw percentages (e.g., 55% vs 73%) without any formal effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or sample size justification is provided. The educator survey has N=76 complete responses out of 209 received, and the developer survey has N=39 (29 complete). No discussion of whether these samples are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Some variance measures are reported: IQRs for participant population sizes (e.g., 'mixed-methods median=52.0 IQR=(24.0-105.0)') and medians for study characteristics in the SLR."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares its findings against prior surveys (e.g., Prather et al. 2023 working group survey with 57 respondents, StackOverflow 2024 developer survey with 46,000 developers) to contextualize results."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Comparisons include the StackOverflow 2024 developer survey (conducted May 2024) and the 2023 ITiCSE working group survey, both of which are recent and relevant."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a survey/SLR paper, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a survey/SLR paper, not a system being evaluated with metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper does not evaluate a system's outputs. It is a survey and literature review."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no test set is involved in this survey/SLR study."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides extensive breakdowns: by tool type vs. outcomes (Table 7), by task type vs. outcomes (Table 8), by tool type and guidance vs. outcomes, and by evidence type vs. findings nature (Table 6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses negative findings from the SLR (Table 6 shows 5 negative and 15 mixed results out of 71 studies), negative perceived outcomes from interviews (academic dishonesty, student unpreparedness), and scenarios where GenAI tools caused harm."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Negative results are reported throughout: 58% of code writing studies reported positive results (meaning 42% did not), hint generation had only 50% positive results, and multiple interview quotes describe negative outcomes including increased cheating and student unpreparedness."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are descriptive and appropriately hedged: it describes summarizing literature, conducting surveys, and interviews. The results sections provide the data supporting these descriptive claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes several implicit causal claims, such as 'when students were not provided guidance and use a general purpose generative AI model, only 55% of studies found positive results' (Section 3.4), implying that guidance or guardrails cause better outcomes. These are cross-tabulations, not causal designs, and the paper does not acknowledge this limitation in interpreting the SLR findings."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to be a 'comprehensive review of current trends' broadly, but the SLR is limited to computing education (not industry or general AI usage). The educator survey has N=76 (predominantly from the US, Germany, Canada) and the developer survey has N=39, yet the paper's conclusions ('Key takeaways') make broad claims like '80% of developers use GenAI tools in their professional roles' without bounding this to the sample."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The threats-to-validity section discusses several alternative explanations: publication bias inflating positive results in the SLR (Section 7.1), self-selection bias in both surveys (Section 7.2), self-reporting bias, and non-representative developer sample. The discussion also notes that educators who haven't explored GenAI may have opinions 'based less on facts than opinions.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper does not use LLMs as part of its methodology. It studies how others use GenAI tools."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no prompting is used in the paper's own methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no model inference is performed as part of this study."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no agentic scaffolding is used in this study's methodology."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The SLR filtering pipeline is well-documented with counts at each stage (1536 → 169 → 71, Figure 1) and explicit exclusion criteria (7 criteria listed in Section 2.2). Survey data filtering is also described: educator survey had 209 responses, 100 removed for incomplete/no consent, 33 test data removed, yielding N=76. Developer survey: 94 → 39 after filtering."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' contains three substantive subsections: 7.1 Systematic Literature Review, 7.2 Educator and Developer Surveys, and 7.3 Interview Study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: SLR limitations include specific database choices, arXiv search being limited to title/abstract, and publication bias. Survey threats include specific mailing list recruitment potentially over-sampling engaged educators, consent form dropout, and the low N=39 for the developer survey. Interview threats include English-only recruitment bias."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While threats to validity are discussed, the paper does not explicitly state what it does NOT claim or what settings are excluded from its conclusions. The 'Key takeaways' box makes broad claims (e.g., '80% of developers use GenAI tools') without explicitly bounding them to the study's sample. The paper does not state e.g., 'these findings apply only to computing education contexts in Western universities' or similar scope restrictions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The SLR data is published on OSF (https://osf.io/wpxjb/). However, the survey and interview data are not mentioned as being publicly available, so this is partial. The SLR data being available earns a YES since it is one of the major data sources."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail for all three methods: SLR search strings, databases, and dates (Section 2); survey distribution channels and dates (Sections 4.5, 4.6); and interview recruitment and process (Sections 4.7, 4.8)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Recruitment channels are explicitly listed for both surveys (Section 4.5: specific mailing lists including ACM SIG, German CS society, ITiCSE participants, LinkedIn, departmental lists) and interviews (Section 4.8: selected based on literature review results, networking at ITiCSE 2024, email to mailing lists, with diversity criteria described)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: SLR from search to final set (Figure 1, 1536→169→71 with criteria at each stage); survey from raw responses to final N (209→76 for educators, 94→39 for developers); and interview coding process (initial joint review, independent tagging, thematic analysis described in Section 4.9)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgments section: Research Council of Finland (Academy Research Fellow grant 356114), U.S. National Science Foundation IUSE Award #2417374, and Google Award for Inclusion Research Program."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are prominently listed at the top of the paper. One author (Jamie Gorson Benario) is affiliated with Google, and Google is also listed as a funder via the 'Google Award for Inclusion Research Program.' All other authors are at academic institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Research Council of Finland, U.S. NSF, and Google Award for Inclusion Research are all independent of the survey outcomes. The paper does not evaluate Google's products specifically, and the funders have no financial stake in particular results about GenAI in education."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. One author works at Google, which produces GenAI products discussed in the paper, but no competing interests declaration addresses this."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper does not evaluate any pre-trained model's capability on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no model benchmarking is performed in this study."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark evaluation is conducted."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for either the surveys or the interview study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "IRB approval is implicitly referenced: Section 4.8.2 states 'To comply with Institutional Review Board (IRB) protocol requirements, these recordings were deleted once the transcriptions were completed.' Consent forms were used for both surveys and interviews."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Demographics are reported for all three samples: educator survey (country, gender, institution type, teaching experience in Tables 10-13), developer survey (country, job title, company type in Table 14), and interview sample (categories: tool creators, educators studying GenAI, educators using GenAI, with selection criteria for diversity in Section 5.1.3)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "For the SLR, seven explicit exclusion criteria are listed in Section 2.2. For the surveys, filtering criteria are described (consent form agreement, completeness). For interviews, three categories of interest are defined with descriptions of who qualifies for each category (Section 4.7)."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is not an experimental study with random assignment. The surveys are cross-sectional and the interviews are purposive sampling."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a survey and interview study, not an experimental study where blinding would be feasible."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Attrition is reported for both surveys: educator survey received 209 responses, 100 dropped (no consent or incomplete), 33 test data removed, yielding N=76. Developer survey received 94 responses, yielding N=39 after filtering, with 29 fully complete. The consent form is identified as a potential cause of dropout (Section 7.2)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a survey/SLR paper, not a system with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a survey/SLR paper with no computational experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "80% of surveyed developers use GenAI tools in their professional software development role.",
       "evidence": "DS-1 results: 31 out of 39 developers (79.5%) reported using GenAI tools (Section 5.7).",
       "supported": "moderate"
     },
     {
       "claim": "Educators underestimate how frequently professional developers use GenAI tools.",
       "evidence": "56.5% of developers report daily GenAI use, while only 30.7% of educators expected daily use (Figure 7, Section 5.7). However, no statistical test is performed on this comparison.",
       "supported": "weak"
     },
     {
       "claim": "Studies using custom tools with pedagogical guardrails report more positive results than those using general-purpose GenAI without guidance.",
       "evidence": "Table 7 shows 73% (19/26) positive results for guardrailed tools without guidance vs. 55% (12/22) for general purpose tools without guidance (Section 3.4).",
       "supported": "moderate"
     },
     {
       "claim": "75% of educators believe the skills to create software have changed after the advent of GenAI.",
       "evidence": "ES-3: 57 out of 76 educators (75%) responded that skills have changed (Section 5.8.1).",
       "supported": "moderate"
     },
     {
       "claim": "Code comprehension tasks show more positive results with GenAI (80%) compared to code writing tasks (58%).",
       "evidence": "Table 8 shows 8/10 (80%) positive for code comprehension vs. 15/26 (58%) positive for writing code (Section 3.4). No statistical test of this difference is reported.",
       "supported": "moderate"
     },
     {
       "claim": "Only 35.5% of educators are actively incorporating GenAI into their courses despite 75% believing skills have changed.",
       "evidence": "ES-2: 27 out of 76 (35.5%) are incorporating GenAI (Section 5.2.1), while ES-3: 57 out of 76 (75%) believe skills have changed (Section 5.8.1).",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "meta-analysis",
     "qualitative",
     "observational"
   ],
   "key_findings": "This working group report combines a systematic literature review of 71 papers with educator (N=76) and developer (N=39) surveys and 17 semi-structured interviews to understand GenAI integration in computing education. The SLR finds that custom tools with pedagogical guardrails produce more positive educational outcomes than unguided general-purpose tools. While 75% of educators believe programming skills are changing due to GenAI (shifting from code writing to code reading, higher-level thinking, and prompt engineering), only 35.5% have actively incorporated GenAI into their courses. Developer GenAI usage (80%) exceeds educator expectations, with a gap between educator perceptions and actual industry practice.",
   "red_flags": [
     {
       "flag": "Very small developer sample",
       "detail": "The developer survey has only N=39 respondents (29 complete), yet the paper's key takeaways present findings as '80% of developers use GenAI tools' without prominent caveats about sample size. Many sub-questions have even smaller N (e.g., N=21 for usefulness ratings, N=18 for country data)."
     },
     {
       "flag": "High survey attrition",
       "detail": "The educator survey received 209 responses but only 76 were usable (63% attrition). 100 responses were dropped due to incomplete data or no consent, plus 33 test responses. This suggests significant self-selection bias among completers."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "The paper makes multiple comparative claims (guardrailed vs. general tools, code comprehension vs. code writing, educator expectations vs. developer reality) based solely on percentage comparisons with no significance tests, confidence intervals, or effect sizes."
     },
     {
       "flag": "Convenience sampling for both surveys",
       "detail": "Both surveys were distributed through author networks, mailing lists, and conference contacts. The developer survey was further limited by the authors' acknowledgment that as 'computing education researchers who do not have access to developers within large tech companies worldwide.' This introduces strong selection bias."
     },
     {
       "flag": "Potential publication bias in SLR not quantified",
       "detail": "The paper acknowledges that publication bias likely inflates positive results in the SLR (Section 7.1) but does not attempt to quantify this bias or perform sensitivity analysis."
     }
   ],
   "cited_papers": [
     {
       "title": "Computing Education in the Era of Generative AI",
       "authors": ["Paul Denny", "James Prather", "Brett A. Becker"],
       "year": 2024,
       "relevance": "Foundational discussion of how GenAI is transforming computing education, providing context for understanding AI tool adoption in programming pedagogy."
     },
     {
       "title": "Studying the Effect of AI Code Generators on Supporting Novice Learners in Introductory Programming",
       "authors": ["Majeed Kazemitabaar"],
       "year": 2023,
       "doi": "10.1145/3544548.3580919",
       "relevance": "Empirical study on how AI code generators affect novice programmers, directly relevant to understanding GenAI impact on learning outcomes."
     },
     {
       "title": "CodeAid: Evaluating a Classroom Deployment of an LLM-based Programming Assistant that Balances Student and Educator Needs",
       "authors": ["Majeed Kazemitabaar"],
       "year": 2024,
       "doi": "10.1145/3613904.3642773",
       "relevance": "Evaluation of an LLM-based programming assistant deployed in a classroom, relevant to understanding agentic AI tools in education."
     },
     {
       "title": "Do Users Write More Insecure Code with AI Assistants?",
       "authors": ["Neil Perry"],
       "year": 2023,
       "relevance": "Empirical study on security implications of AI code assistants, relevant to evaluating risks and quality of AI-generated code."
     },
     {
       "title": "Lost at C: A User Study on the Security Implications of Large Language Model Code Assistants",
       "authors": ["Gustavo Sandoval"],
       "year": 2023,
       "relevance": "User study examining security implications of LLM code assistants, relevant to AI code generation safety evaluation."
     },
     {
       "title": "CodeHelp: Using Large Language Models with Guardrails for Scalable Support in Programming Classes",
       "authors": ["Mark Liffiton", "Brad E Sheese", "Jaromir Savelka", "Paul Denny"],
       "year": 2024,
       "doi": "10.1145/3631802.3631830",
       "relevance": "LLM-based programming tool with pedagogical guardrails, directly relevant to understanding safety and quality of AI-assisted coding tools."
     },
     {
       "title": "Grounded Copilot: How Programmers Interact with Code-Generating Models",
       "authors": ["Shraddha Barke", "Michael B James", "Nadia Polikarpova"],
       "year": 2023,
       "doi": "10.1145/3586030",
       "relevance": "Empirical study of programmer interaction with AI code generation tools, relevant to understanding productivity and workflow impacts."
     },
     {
       "title": "A Large-Scale Survey on the Usability of AI Programming Assistants: Successes and Challenges",
       "authors": ["Jenny T. Liang", "Chenyang Yang", "Brad A. Myers"],
       "year": 2024,
       "doi": "10.1145/3597503.3608128",
       "relevance": "Large-scale survey on AI programming assistant usability, providing complementary data on developer experience with AI coding tools."
     },
     {
       "title": "It's Weird That it Knows What I Want: Usability and Interactions with Copilot for Novice Programmers",
       "authors": ["James Prather"],
       "year": 2023,
       "relevance": "Study on novice programmer interactions with GitHub Copilot, relevant to understanding how AI assistants affect programming learning."
     },
     {
       "title": "How Far Are We? The Triumphs and Trials of Generative AI in Learning Software Engineering",
       "authors": ["Rudrajit Choudhuri"],
       "year": 2024,
       "doi": "10.1145/3597503.3639201",
       "relevance": "Empirical investigation of GenAI effectiveness in software engineering education, directly relevant to the survey scope."
     },
     {
       "title": "How Beginning Programmers and Code LLMs (Mis)read Each Other",
       "authors": ["Sydney Nguyen"],
       "year": 2024,
       "doi": "10.1145/3613904.3642706",
       "relevance": "Study on mismatches between novice programmers and LLMs, relevant to understanding challenges in AI-assisted programming."
     }
   ]
 }