ai-research-survey

scan.json (28680B)

---

 {
   "paper": {
     "title": "AI-Tutoring in Software Engineering Education: Experiences with Large Language Models in Programming Assessments",
     "authors": [
       "Eduard Frankford",
       "Clemens Sauerwein",
       "Patrick Bassner",
       "Stephan Krusche",
       "Ruth Breu"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2404.02548",
     "doi": ""
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides Artemis integration files on Figshare (https://figshare.com/s/636a9c5ff8f2c8315f26) as described in Section 8 Data Availability. However, a comprehensive repository was not released 'due to challenges associated with its anonymization.' The Artemis platform itself is open source on GitHub (https://github.com/ls1intum/Artemis). The Figshare archive with integration files counts as a code release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 describes the dataset available on Figshare including: anonymized data analysis spreadsheet with extracted database data, survey responses, code analysis, and student submissions to the version control system. All personal identifiers were removed."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. There is no requirements.txt, Dockerfile, or detailed environment setup section. The paper mentions the Artemis platform and GPT-3.5-Turbo API but does not specify software versions, library dependencies, or setup instructions beyond pointing to the open-source Artemis project."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Section 8 describes the released data and Artemis integration files but does not include a README with commands, a 'Reproducing Results' section, or scripts to replicate the study. A researcher would need to reverse-engineer the integration from the released files."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports Likert scale averages (e.g., 'Somewhat Agree (1.29)', 'Neutral (-0.29)') without confidence intervals or error bars. The stacked bar chart in Figure 5 shows response distributions but no uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative observations (e.g., different user personas, feedback quality categories) but does not use any statistical significance tests. The authors acknowledge the sample is too small for quantitative analysis (Section 6.3) but still report numerical averages without testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. The paper presents raw percentages (e.g., '66.6% were useful') and Likert averages without formal effect size measures like Cohen's d or relative comparisons with baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 acknowledges the modest sample size (23 students, 12 active) and justifies the qualitative approach: 'While this may seem like a modest sample size, it's important to note that the qualitative nature of this analysis allowed for a more in-depth understanding of individual experiences.' Section 6.3 further notes it is 'too small to conduct a statistically significant quantitative analysis.'"
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported. Likert responses are summarized as averages without spread measures. The stacked bar chart shows distributions visually but no numeric variance is provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baseline comparison is included. The study does not compare the AI-Tutor against a control group (no AI-Tutor), against human tutors, or against any alternative system. Students could optionally use the AI-Tutor, but there is no systematic comparison of outcomes with vs. without it."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included at all, so the question of whether they are contemporary does not arise. The paper only reports on the AI-Tutor's integration without comparison to any alternative."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The system has essentially one component (GPT-3.5-Turbo with a single prompt template). There are no modular components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple evaluation approaches: (1) interaction pattern analysis with temporal coding, (2) TAM survey with 6 Likert items, (3) open-ended qualitative survey questions, and (4) manual feedback quality categorization (useful vs. not useful, with subcategories). These constitute multiple metrics for different research questions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The study includes a survey of students evaluating their experience with the AI-Tutor's outputs (TAM questions and open-ended feedback). Additionally, the researchers manually evaluated 75 feedback responses for quality (Section 4.3: '55(66.6%) were useful and 20(26.6%) were categorized as not useful')."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a case study with qualitative analysis, not a machine learning evaluation. There is no concept of a held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides breakdowns: interaction patterns are categorized into two user personas (Iterative Ivy, Hybrid Harry). Among 20 not-useful responses, breakdown is given (3 revealed solutions, 4 hallucinations, 13 too general). TAM responses are reported per question. Figure 4 shows per-student interaction timelines."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses failure cases in detail: 20 out of 75 feedback responses were not useful, categorized as solution-revealing (3), hallucinations (4), and too general (13). Specific examples of hallucination types are described (e.g., LLM stating a function looks well implemented when no implementation existed)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: neutral-to-negative TAM responses on most questions, 26.6% feedback being not useful, students' concerns about learning inhibition, generic feedback quality, hallucination issues, and API dependency problems. Section 4.2 honestly reports the mixed/neutral survey results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are modest: 'identified different user types based on their interaction patterns' (supported in Section 4.1), 'advantages, such as timely feedback and scalability' (supported in Section 4.3), 'challenges like generic responses and students' concerns about a learning progress inhibition' (supported in Sections 4.2-4.3). No overclaiming is present."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims, e.g., 'This immediate feedback helped students to quickly correct their errors' (Section 4.3), 'the AI-Tutor, in its capacity, guides through specific instructions' implying tutoring causes improvement. There is no control group or causal identification strategy; the observational design cannot establish that the AI-Tutor caused any learning improvements."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper overgeneralizes in several places. Section 6.3 claims findings from GPT-3.5-Turbo 'can largely be extrapolated to other similar models' without evidence. Section 4.3 suggests applicability to MOOCs based on a single exercise with 23 students. The title 'AI-Tutoring in Software Engineering Education' is broader than the actual scope (one C programming exercise at one university)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section (Section 6) discusses generic methodological concerns but does not consider specific alternative explanations for findings. For example, students preferring the AI-Tutor might simply value novelty; the 'Iterative Ivy' pattern could reflect students who are generally more cautious, not AI-Tutor effectiveness. No such alternatives are discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'GPT-3.5-Turbo' throughout but never specifies a snapshot date or API version (e.g., 'gpt-3.5-turbo-0613'). Model behavior changes across versions, and the specific version used is not identified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full prompt template is provided in Listing 1 (Section 3.1). The placeholders are clearly explained: 'language' is English or German, 'description' is the Pascal's triangle task description, 'current' is the student's code, and 'solution' is the instructor's solution. The actual prompt structure with fill values is sufficiently described for reconstruction."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is reported as 0.7 (Section 3.1), but other hyperparameters such as top-p, max tokens, and frequency/presence penalties are not mentioned. The paper only states temperature, which is insufficient for full reproducibility."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The system architecture is described in detail in Section 3.1 with a sequence diagram (Figure 1). The workflow is: student presses button -> server retrieves student code, exercise description, and sample solution -> API call to OpenAI -> response displayed in popup. No retry logic, memory, or multi-turn interaction is used (by design). The scaffolding is simple and fully described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 documents data analysis: combination of two datasets (AI Feedback and student submissions), exclusion of students who did not solve the exercise or engage with the AI-Tutor (reducing from 23 to 12 active participants), and the three-phase qualitative analysis approach (temporal coding, thematic coding, theme development)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations' is a dedicated section with four subsections: Construct Validity (6.1), Reliability (6.2), External Validity (6.3), and Internal Validity (6.4). This is a substantive multi-page discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Several specific threats are discussed: the non-deterministic nature of GPT-3.5-Turbo affecting reliability (Section 6.2), integration into only one APAS (Artemis) limiting generalizability (Section 6.3), sample size of 23 being too small for quantitative analysis (Section 6.3), and use of a single LLM (Section 6.3). These are specific to this study rather than generic boilerplate."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the limitations section discusses threats to validity, it does not explicitly state what the results do NOT show. The paper does not clearly bound the scope to say, for example, 'these results do not show that AI-Tutors improve learning outcomes' or 'we do not claim effectiveness beyond introductory C programming.' Instead, claims like findings being 'extrapolated to other similar models' (Section 6.3) expand rather than bound the scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 describes the data availability on Figshare including anonymized raw data: database extracts with code/feedback/user/timestamps, survey responses, student code submissions, and code analysis. This allows independent verification of the qualitative findings."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection in detail: the APAS integration stores code, feedback, user, file, and timestamp on each AI feedback request (Section 3.1). Student submissions with test results and timestamps are stored in the version control system. The survey was administered in a 15-minute session in the following tutorial (Section 3.2)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes recruitment: participants were students from the 'Introduction to Programming' tutorial at the University of Innsbruck, part of the Bachelor in Computer Science curriculum. 23 students actively participated. They were introduced to the AI-Tutor functionality before the exercise."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 documents the pipeline: two datasets were combined (AI Feedback data and student submissions), students who didn't engage were excluded (from 23 to 12 active), and the three-stage qualitative analysis was applied (temporal coding, thematic coding, theme development). The filtering rationale and stages are explained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 Acknowledgments states: 'The CodeAbility Austria project has been funded by the Austrian Federal Ministry of Education, Science and Research (BMBWF).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Eduard Frankford, Clemens Sauerwein, and Ruth Breu at University of Innsbruck; Patrick Bassner and Stephan Krusche at Technical University of Munich. The Artemis platform being evaluated was developed at TU Munich (co-authors Bassner and Krusche are affiliated there), which represents a potential conflict that is implicitly visible through affiliations."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is the Austrian Federal Ministry of Education, Science and Research (BMBWF), a government agency with no apparent financial stake in whether GPT-3.5-Turbo works as an AI-Tutor or whether Artemis is effective."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial interests statement in the paper. Stephan Krusche is the creator/lead of the Artemis platform being evaluated, which represents a potential interest, but no declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on a benchmark. It uses GPT-3.5-Turbo as a tutoring tool and evaluates the student experience, not the model's knowledge against a test set."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. The study evaluates the AI-Tutor's tutoring effectiveness with students, not model performance on a benchmark. There is no train/test overlap concern."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. The study is not a benchmark evaluation of model capabilities. Pascal's Triangle is the exercise students solve, not a benchmark for measuring LLM performance."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration found in the paper. No link to OSF, AsPredicted, or any other pre-registration repository is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics board approval found in the paper. The study collected data from student participants (survey responses, interaction data) without stating ethics review."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 states the survey asked about demographics including 'their highest degree, their current semester and their programming experience.' The students are characterized as first-year Bachelor of Computer Science students at the University of Innsbruck in the 'Introduction to Programming' course using C."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 specifies the population: students from the 'Introduction to Programming' tutorial at the University of Innsbruck. Section 3.3 describes exclusion: 'students who did not solve the exercise and did not engage with the AI-Tutor were excluded from the qualitative analysis' (reducing to 12 participants)."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational/exploratory case study, not an experimental study with treatment and control conditions. Students chose voluntarily whether to use the AI-Tutor. No randomization was applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. This is an observational study, not an experiment with conditions. Students knew they were using the AI-Tutor; blinding is not feasible in this design."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Attrition is reported: 23 students participated, but only 12 'worked intensively with both systems' and were included in the qualitative analysis (Section 3.3, 4.1). The paper details various categories of non-participation: 4 made no submissions and no AI requests, 1 submitted without AI, 1 used AI without submitting, 2 submitted without AI, etc."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, cost per feedback request, or total expenditure is reported despite using a commercial API (OpenAI GPT-3.5-Turbo). The paper discusses resource efficiency abstractly as a design consideration (Section 3.1) but provides no actual cost data."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or token consumption is reported. The paper mentions 75 feedback requests were made but does not quantify the tokens consumed or costs incurred."
       }
     }
   },
   "claims": [
     {
       "claim": "Two distinct user personas emerged: 'Iterative Ivy' (continuous AI feedback before submission) and 'Hybrid Harry' (alternating between AI feedback and APAS submissions).",
       "evidence": "Section 4.1 with Figure 4 showing temporal interaction patterns for 12 active students. Detailed descriptions of each persona's behavior in Sections 4.1.1 and 4.1.2.",
       "supported": "moderate"
     },
     {
       "claim": "Student satisfaction with the AI-Tutor was overall neutral, with mixed positive and negative responses that 'neutralize one another.'",
       "evidence": "Section 4.2 with Figure 5 showing stacked bar distributions of Likert responses. Average sentiment scores ranging from -0.43 to 1.29, with only 'easy to use' receiving a non-neutral average.",
       "supported": "moderate"
     },
     {
       "claim": "66.6% (55 out of 75) of AI-Tutor feedback responses were useful, while 26.6% (20) were not useful.",
       "evidence": "Section 4.3 states this directly with subcategorization: 3 revealed solutions, 4 were hallucinations, 13 were too general. Manual review of each feedback instance.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-3.5-Turbo can be successfully used as a language model behind an AI-Tutor.",
       "evidence": "Section 5 Discussion. Based on the system functioning and receiving mixed-neutral survey responses. The claim is partially undermined by the 26.6% not-useful rate and neutral TAM results.",
       "supported": "weak"
     },
     {
       "claim": "The AI-Tutor was able to give feedback on logical and semantic issues, such as incorrect loop boundary conditions.",
       "evidence": "Section 4.3 describes this observation qualitatively: 'if students had defined wrong boundary conditions to terminate a loop, then the AI-Tutor recognized this and proposed to the student to change this condition.'",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "case-study",
     "qualitative"
   ],
   "key_findings": "An exploratory case study integrating GPT-3.5-Turbo as an AI-Tutor in the Artemis APAS with 23 students (12 active) found two interaction personas: students who used AI feedback iteratively before submitting, and those who alternated between AI feedback and system submissions. Student satisfaction was overall neutral on TAM measures, with 66.6% of AI feedback rated useful by manual review, while 26.6% was too generic, hallucinated, or revealed solutions. The study highlights both the potential for scalable real-time feedback and challenges including generic responses, hallucinations, and student concerns about over-reliance inhibiting learning progress.",
   "red_flags": [
     {
       "flag": "No control group",
       "detail": "The study has no baseline comparison. Students who used the AI-Tutor are not compared against students who did not, making it impossible to determine whether the AI-Tutor had any effect on learning outcomes or task completion."
     },
     {
       "flag": "Tiny sample with quantitative reporting",
       "detail": "With only 23 students (12 active), the paper reports numerical averages of Likert responses (e.g., 1.29, -0.29) without uncertainty measures. These numbers are meaningless at this sample size without confidence intervals, yet they are presented as findings."
     },
     {
       "flag": "Overgeneralization from single exercise",
       "detail": "The study uses a single programming exercise (Pascal's Triangle in C) at one university, yet makes claims about AI-Tutoring in 'Software Engineering Education' broadly and suggests applicability to MOOCs and other LLMs."
     },
     {
       "flag": "Conflict of interest not acknowledged",
       "detail": "Co-authors Bassner and Krusche are from TU Munich and are involved in developing the Artemis platform being evaluated. Krusche is the original creator of Artemis. No competing interests statement is present."
     },
     {
       "flag": "No ethics approval reported",
       "detail": "The study collected interaction data and survey responses from students (a vulnerable population in an instructor-student power dynamic) without mentioning IRB or ethics board approval."
     },
     {
       "flag": "Selection bias in qualitative analysis",
       "detail": "The qualitative analysis excludes 11 of 23 students, focusing only on the 12 who actively used both the AI-Tutor and APAS. This selection bias means the analysis only captures experiences of students who chose to engage, not the broader population."
     }
   ],
   "cited_papers": [
     {
       "title": "How ChatGPT Will Change Software Engineering Education",
       "authors": ["Marian Daun", "Jennifer Brings"],
       "year": 2023,
       "relevance": "Directly addresses the impact of ChatGPT on software engineering education, relevant to understanding how LLMs are changing programming pedagogy."
     },
     {
       "title": "ChatGPT for good? On opportunities and challenges of large language models for education",
       "authors": ["Enkelejda Kasneci", "Kathrin Sessler", "Stefan Küchemann"],
       "year": 2023,
       "doi": "10.1016/j.lindif.2023.102274",
       "relevance": "Discusses opportunities and challenges of LLMs in education, relevant to understanding the broader context of AI-assisted learning."
     },
     {
       "title": "An analysis of the automatic bug fixing performance of chatgpt",
       "authors": ["Dominik Sobania", "Martin Briesch", "Carol Hanna", "Justyna Petke"],
       "year": 2023,
       "arxiv_id": "2301.08653",
       "relevance": "Evaluates ChatGPT's bug-fixing capability on QuixBugs benchmark, relevant to assessing LLM capability in code repair."
     },
     {
       "title": "Is ChatGPT the Ultimate Programming Assistant–How far is it?",
       "authors": ["Haoye Tian", "Weiqi Lu", "Tsz On Li"],
       "year": 2023,
       "arxiv_id": "2304.11938",
       "relevance": "Evaluates ChatGPT as a programming assistant for code generation, repair, and summarization, directly relevant to LLM programming capability assessment."
     },
     {
       "title": "ChatGPT, Can You Generate Solutions for my Coding Exercises? An Evaluation on its Effectiveness in an undergraduate Java Programming Course",
       "authors": ["Eng Lieh Ouh", "Benjamin Kok Siew Gan", "Kyong Jin Shim", "Swavek Wlodkowski"],
       "year": 2023,
       "arxiv_id": "2305.13680",
       "relevance": "Empirical analysis of ChatGPT's potential as a programming assistant in an educational context."
     },
     {
       "title": "Jailbreaking chatgpt via prompt engineering: An empirical study",
       "authors": ["Yi Liu", "Gelei Deng", "Zhengzi Xu"],
       "year": 2023,
       "arxiv_id": "2305.13860",
       "relevance": "Studies prompt engineering attacks on ChatGPT, relevant to AI safety and understanding vulnerabilities in LLM-based educational tools."
     },
     {
       "title": "Learning gain differences between ChatGPT and human tutor generated algebra hints",
       "authors": ["Zachary A Pardos", "Shreya Bhandari"],
       "year": 2023,
       "arxiv_id": "2302.06871",
       "relevance": "Compares ChatGPT-generated vs human tutor hints for learning gain, directly relevant to evaluating LLM tutoring effectiveness."
     },
     {
       "title": "Chatgpt and software testing education: Promises & perils",
       "authors": ["Sajed Jalil", "Suzzana Rafi", "Thomas D LaToza", "Kevin Moran", "Wing Lam"],
       "year": 2023,
       "relevance": "Evaluates ChatGPT's performance in software testing education, relevant to understanding LLM capability in SE domains."
     },
     {
       "title": "What is the impact of ChatGPT on education? A rapid review of the literature",
       "authors": ["Chung Kwan Lo"],
       "year": 2023,
       "relevance": "Literature review of ChatGPT's educational impact, relevant as a survey of LLM applications in education."
     }
   ]
 }