ai-research-survey

scan.json (24136B)

---

 {
   "paper": {
     "title": "Using Assignment Incentives to Reduce Student Procrastination and Encourage Code Review Interactions",
     "authors": ["Kevin Wang", "Ramon Lawrence"],
     "year": 2023,
     "venue": "2023 International Conference on Computational Science and Computational Intelligence (CSCI)",
     "arxiv_id": "2311.15125",
     "doi": "10.1109/CSCI62032.2023.00270"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The authors provide a GitHub link to their assignments and instructions: 'github.com/rlawrenc/cosc_404/tree/404_2023_Jan/labs' (footnote 1, Section II.D). This is the course material, not analysis code, but it is the primary artifact of the intervention."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset is released. The queue system data, survey responses, and grade data are not made publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided. The paper does not describe any software dependencies, analysis tools, or computational environment used for the data analysis."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided for replicating the analysis. The paper describes the intervention design but not how to reproduce the quantitative analysis."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for any of the main results. The paper reports point estimates only (e.g., '45% of assignments completed early', '78% overall exam average')."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'The performance on the final exam for students that had at least one early marked assignment was statistically significant' (Section V.D) but does not report which test was used, the test statistic, or a p-value. This is insufficient — claiming statistical significance without reporting the test or p-value does not satisfy this criterion."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports exam averages with baseline context: 'the final exam average was 78% for all students. Students that had no bonus marked assignment had an average of 71% versus 81% for students with at least 1 bonus marked assignment' (Section V.D). This provides the magnitude of the effect (10 percentage point difference) with baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the sample size (107 students, 67 consenting, 44 survey respondents). No power analysis is discussed. The 41% survey response rate is noted but not discussed as a limitation."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported for any of the quantitative results (exam scores, help session times, etc.). Only means and percentages are given."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'significant student acceptance and behavior change compared to previous course offerings' (Section V.A) but provides no quantitative baseline data from previous offerings. No formal comparison against a control group or historical baseline is presented."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No quantitative baselines are provided, so contemporariness cannot be assessed. The paper references prior course offerings qualitatively but does not present data from them."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The intervention combines bonus marks with synchronous code review, but no ablation separates their individual effects. It is unclear whether the behavior change is due to bonus marks, code review, or their combination."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple metrics: early submission rates (45%), survey satisfaction (92% agree/strongly agree), help session question distribution (Figure 1), questions by day (Figure 2), general questions vs marked assignments (Figure 3), and final exam averages."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The study includes a student survey (N=44) evaluating opinions on the incentive system, and interviews with instructional staff were conducted. These constitute human evaluation of the intervention."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning study with train/test splits. The concept of a held-out test set does not apply to this pedagogical intervention study."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides breakdowns by day of early submission (70% Monday, 7% Tuesday, 14% Wednesday, etc.), by question type (Figure 1), by number of marked assignments vs general questions (Figure 3), and by exam performance grouped by bonus assignment count (Figure 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses practical challenges: 'Assignment incentivization takes help session time that may be required for general assignment help questions' and 'If the burden of grading is too high, wait times become high for all students and the approach would not be practical.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper honestly reports that the correlation between early submission and exam scores may be due to confounding: 'the lower average for students with fewer bonus marked assignments could be attributed to the original ability level and intrinsic motivation of the student' (Section V.D). The paper also notes 'there is no time savings' for grading (Section V.C)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '45% of assignments completed early and 30% up to 4 days before the deadline' — the results section reports 234/523 (45%) early submissions and 70% of those on Monday (4 days before deadline). The abstract claim of 'no increase in marking time' is supported in Section V.C. All abstract claims are substantiated."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'assignment incentivization practice of awarding bonus marks for early submission effectively curbs procrastination' (Section VI), but the study design is observational (single course, no control group, no randomization). The paper acknowledges this partially for exam performance but not for the procrastination claim itself. Confounders like course content differences, student cohort differences, and instructor effects are not controlled for."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds generalizations: 'The research was conducted in an upper-level course with well-defined assignments that were verified for correctness using unit tests. The results may be generalizable to other courses as long as assignment correctness can be verified quickly' (Section VI). It specifies conditions under which the technique may or may not transfer."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses the selection bias alternative explanation: 'the lower average for students with fewer bonus marked assignments could be attributed to the original ability level and intrinsic motivation of the student, as more capable and motivated students with higher test scores are also more likely to complete assignments early' (Section V.D)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not use any AI/ML models. It is a pedagogical intervention study and does not involve LLMs or other pre-trained models."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in this study. It is a pedagogical intervention study, not an AI/LLM study."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No models or algorithms with hyperparameters are used. This is a pedagogical intervention study."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a pedagogical intervention study, not an AI agent study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes data preprocessing: 'Times less than 30 seconds are eliminated as these often occur where no help was given' (Section IV). It also describes which students were included: 'Help session data collected was analyzed for students that used the HelpMe queue system (N=83)' and '67 students (62%) consented to analysis of their grades.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Some limitations are discussed within the Discussion section (Section VI), but there is no substantive dedicated subsection."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: the confound between student ability and early submission ('could be attributed to the original ability level and intrinsic motivation'), scalability concerns ('If the burden of grading is too high, wait times become high'), and practical deployment constraints. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states explicit scope boundaries: 'The research was conducted in an upper-level course with well-defined assignments that were verified for correctness using unit tests' and notes 'A more detailed study would be required to determine if the student learning was positively impacted' (Section V.D)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data (queue system logs, survey responses, exam grades) is not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV describes data collection: 'Data on student interactions and assignment completion was collected from an online queue system, which provides details regarding instructor-student interactions including wait and help session times, types of questions, time of day when questions were asked, and user information. The second set of quantitative data came from the exam and overall grades of consenting participants.'"
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Participants were students enrolled in the course — this is clearly stated: 'The course had 99 undergraduate students and 8 graduate students' (Section II.D). The survey was provided in the last two weeks of class. The participation pathway is clear (course enrollment), though selection bias in survey response (41% rate) is not discussed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents the pipeline: 107 registered students → 67 consented to grade analysis (62%) → 83 used the queue system → 44 survey responses (41% response rate). Filtering criteria for help session times (>30 seconds) are specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or disclosure is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: both authors are from the Department of Computer Science, University of British Columbia. One author (Ramon Lawrence) appears to be the course instructor, which is relevant as the evaluator of their own intervention."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence of funding cannot be assessed. Absence of disclosure is treated as NO."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate any pre-trained model on a benchmark. It is a pedagogical intervention study."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate any pre-trained model on a benchmark. It is a pedagogical intervention study."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate any pre-trained model on a benchmark. It is a pedagogical intervention study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration. No link to OSF, AsPredicted, or any registry."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'This research uses both survey and quantitative data collected and processed in accordance with a university approved ethics study' (Section IV)."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Some demographics are reported: '99 undergraduate students and 8 graduate students', 'All students are third year or above', 'the majority of students in the course were computer science majors' (Section II.D). Gender, geographic distribution, and programming experience are not reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Inclusion criteria are implicit but clear: all students enrolled in the upper-level database systems course. For grade analysis, '67 students (62%) consented'. For help sessions, 'students that used the HelpMe queue system (N=83)'. Filtering criteria are specified."
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "There is no randomization. All students in the course received the same incentive treatment. There is no control group or random assignment to conditions. This is an observational study of a whole-class intervention."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Blinding is not feasible in this study design. Students necessarily know they are receiving bonus marks for early submission. This is a classroom intervention, not an experiment where blinding would be applicable."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Attrition information is provided: 107 registered students, 67 consented to grade analysis, 83 used the queue system, 44 responded to the survey (41% response rate). The different N values across analyses are reported."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a pedagogical intervention study, not an AI/ML system. Inference cost does not apply."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a pedagogical intervention study, not a computational experiment. Compute budget does not apply."
       }
     }
   },
   "claims": [
     {
       "claim": "45% of assignments (234 of 523) were submitted early through help sessions",
       "evidence": "Section V.A: '234 of the total 523 assignment submissions (45%) were done in help sessions.' Breakdown by day: 70% Monday, 7% Tuesday, 14% Wednesday, 7% Thursday, 2% Friday.",
       "supported": "strong"
     },
     {
       "claim": "92% of surveyed students agree or strongly agree they are motivated to complete assignments earlier",
       "evidence": "Table I: 82% strongly agree + 10% agree = 92% for the question about bonus marks motivating earlier completion. Survey N=44 (41% response rate).",
       "supported": "moderate"
     },
     {
       "claim": "Students with at least one bonus marked assignment had higher final exam scores (81% vs 71%)",
       "evidence": "Section V.D: 'Overall, the final exam average was 78% for all students. Students that had no bonus marked assignment had an average of 71% versus 81% for students with at least 1 bonus marked assignment.'",
       "supported": "weak"
     },
     {
       "claim": "The incentive system shifts help session demand from right before deadlines to Monday bonus deadlines",
       "evidence": "Section V.B and Figure 2: 'the number of help sessions is small on the Friday (F) due date, and instead significant peaks are visible on the Monday (M), when the 10% bonus mark is provided.'",
       "supported": "moderate"
     },
     {
       "claim": "Synchronous grading during office hours results in more detailed feedback with no increase in marking time",
       "evidence": "Section V.C: Traditional grading takes about 5 minutes; virtual office hours grading takes 3-5 minutes but can extend to 10+ minutes. 'Overall, there is no time savings, but two TAs were able to support a class of over a hundred students while providing personalized, real-time feedback.'",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["observational", "qualitative"],
   "key_findings": "An assignment incentive system offering 5-10% bonus marks for early submission combined with synchronous code review shifted student submission behavior, with 45% of assignments completed early and 70% of those on the Monday bonus deadline (4 days before the due date). Survey results (N=44, 41% response rate) showed 92% of respondents agreed the incentives motivated earlier completion. Students with early submissions had higher exam averages (81% vs 71%), though the authors acknowledge this correlation may be driven by pre-existing student ability differences rather than the intervention itself.",
   "red_flags": [
     {
       "flag": "No control group",
       "detail": "The study applies the intervention to all students in one course offering with no control group. There is no quantitative comparison to a prior semester or untreated group, making it impossible to attribute behavior changes to the intervention versus other factors (different cohort, different semester, instructor effects)."
     },
     {
       "flag": "Low survey response rate with likely self-selection bias",
       "detail": "Only 44 of 107 students (41%) responded to the survey. Students who benefited from or engaged with the incentive system are likely overrepresented among respondents. The 92% satisfaction figure may not represent the full class."
     },
     {
       "flag": "Statistical significance claimed without reporting the test",
       "detail": "Section V.D claims 'The performance on the final exam for students that had at least one early marked assignment was statistically significant' without naming the test, reporting the test statistic, or providing a p-value."
     },
     {
       "flag": "Confounding between ability and treatment",
       "detail": "The paper acknowledges that higher-performing students may self-select into early submission. Without randomization or controls for prior ability, the exam performance difference (81% vs 71%) cannot be attributed to the incentive system."
     },
     {
       "flag": "Instructor evaluating own intervention",
       "detail": "The authors designed and implemented the incentive system in their own course and then evaluated it. While not inherently disqualifying, this creates potential bias in how results are interpreted and reported."
     }
   ],
   "cited_papers": [
     {
       "title": "Does the Early Bird Catch the Worm? Earliness of Students' Work and its Relationship with Course Outcomes",
       "authors": ["J. Leinonen", "F. E. V. Castro", "A. Hellas"],
       "year": 2021,
       "doi": "10.1145/3430665.3456383",
       "relevance": "Empirical study of relationship between early assignment start times and course performance in CS education, relevant to understanding AI-assisted completion incentives."
     },
     {
       "title": "Context-Aware and Data-Driven Feedback Generation for Programming Assignments",
       "authors": ["D. Song", "W. Lee", "H. Oh"],
       "year": 2021,
       "doi": "10.1145/3468264.3468598",
       "relevance": "AI-driven automated feedback for programming assignments, relevant to understanding automated code review and AI-assisted programming education."
     },
     {
       "title": "Applying Gamification to Motivate Students to Write High-Quality Code in Programming Assignments",
       "authors": ["R. Kasahara", "K. Sakamoto", "H. Washizaki", "Y. Fukazawa"],
       "year": 2019,
       "doi": "10.1145/3304221.3319792",
       "relevance": "Gamification approach to improving code quality in assignments, relevant to understanding incentive structures in programming education."
     },
     {
       "title": "Promoting Early Engagement with Programming Assignments Using Scheduled Automated Feedback",
       "authors": ["P. Denny", "J. Whalley", "J. Leinonen"],
       "year": 2021,
       "doi": "10.1145/3441636.3442309",
       "relevance": "Automated feedback system for encouraging early engagement with programming, relevant to understanding AI-assisted code review approaches."
     },
     {
       "title": "HelpMe: Student Help Seeking using Office Hours and Email",
       "authors": ["K. Wang", "R. Lawrence"],
       "year": 2024,
       "doi": "10.1145/3626252.3630867",
       "relevance": "The online queue system used in this study for tracking student-instructor interactions, relevant to infrastructure for studying AI-assisted programming education."
     }
   ]
 }