scan.json (14442B)
1 { 2 "paper": { 3 "title": "Reshaping higher education in the unavoidable era of AI", 4 "authors": ["Ahmet Baytak"], 5 "year": 2025, 6 "venue": "16th London International Conference", 7 "doi": "10.31039/plic.2025.14.338" 8 }, 9 "scan_version": 2, 10 "active_modules": [], 11 "methodology_tags": ["theoretical"], 12 "key_findings": "This paper argues that AI integration in higher education requires a fundamental shift from traditional pedagogy toward cultivating uniquely human skills. It distinguishes between 'AI Integrated Education' (AI as assistant) and 'AI Leading Education' (AI as central pillar), advocating for the latter. The paper warns that over-reliance on AI tools weakens synthesis, critical thinking, and problem-solving skills, and proposes process-oriented assessment strategies to address AI-generated plagiarism.", 13 "checklist": { 14 "artifacts": { 15 "code_released": { 16 "applies": false, 17 "answer": false, 18 "justification": "Theoretical/position paper with no code or system to release." 19 }, 20 "data_released": { 21 "applies": false, 22 "answer": false, 23 "justification": "No data collected or analyzed in this paper." 24 }, 25 "environment_specified": { 26 "applies": false, 27 "answer": false, 28 "justification": "No computational experiments performed." 29 }, 30 "reproduction_instructions": { 31 "applies": false, 32 "answer": false, 33 "justification": "No experiments to reproduce; this is a theoretical commentary." 34 } 35 }, 36 "statistical_methodology": { 37 "confidence_intervals_or_error_bars": { 38 "applies": false, 39 "answer": false, 40 "justification": "No quantitative analysis or experiments performed." 41 }, 42 "significance_tests": { 43 "applies": false, 44 "answer": false, 45 "justification": "No comparative claims based on the authors' own data." 46 }, 47 "effect_sizes_reported": { 48 "applies": false, 49 "answer": false, 50 "justification": "No experiments or quantitative analysis." 51 }, 52 "sample_size_justified": { 53 "applies": false, 54 "answer": false, 55 "justification": "Theoretical paper with no sample." 56 }, 57 "variance_reported": { 58 "applies": false, 59 "answer": false, 60 "justification": "No experimental runs." 61 } 62 }, 63 "evaluation_design": { 64 "baselines_included": { 65 "applies": false, 66 "answer": false, 67 "justification": "No evaluation or experiment conducted." 68 }, 69 "baselines_contemporary": { 70 "applies": false, 71 "answer": false, 72 "justification": "No evaluation conducted." 73 }, 74 "ablation_study": { 75 "applies": false, 76 "answer": false, 77 "justification": "No system or components to ablate." 78 }, 79 "multiple_metrics": { 80 "applies": false, 81 "answer": false, 82 "justification": "No metrics used; theoretical paper." 83 }, 84 "human_evaluation": { 85 "applies": false, 86 "answer": false, 87 "justification": "No system outputs to evaluate." 88 }, 89 "held_out_test_set": { 90 "applies": false, 91 "answer": false, 92 "justification": "No data or evaluation." 93 }, 94 "per_category_breakdown": { 95 "applies": false, 96 "answer": false, 97 "justification": "No quantitative results to break down." 98 }, 99 "failure_cases_discussed": { 100 "applies": true, 101 "answer": true, 102 "justification": "The paper discusses failure modes of AI in education: over-reliance weakening cognitive skills, the 19% productivity penalty for experienced developers (citing Klang & Fuller 2025), and AI-generated text evading plagiarism detection." 103 }, 104 "negative_results_reported": { 105 "applies": true, 106 "answer": true, 107 "justification": "The paper cites negative findings: experienced developers took 19% longer with AI (Klang & Fuller 2025), and AI study tools didn't greatly change medical students' test scores (Bharat & Smith 2025)." 108 } 109 }, 110 "claims_and_evidence": { 111 "abstract_claims_supported": { 112 "applies": true, 113 "answer": true, 114 "justification": "The abstract claims AI is reshaping higher education and that institutions need to adapt. The body explores these themes with cited examples. Claims are broad and qualitative, matching the paper's scope." 115 }, 116 "causal_claims_justified": { 117 "applies": true, 118 "answer": false, 119 "justification": "The paper makes causal claims like 'overreliance on AI tools poses a significant threat to the development of essential cognitive skills' and 'consistent reliance on AI can lead to an atrophy of our capacity for deep analysis.' These are presented as established facts with only general citations (Dwivedi et al. 2023, Carr 2020) rather than causal evidence from controlled studies." 120 }, 121 "generalization_bounded": { 122 "applies": true, 123 "answer": false, 124 "justification": "The paper makes sweeping claims about all of higher education without bounding to specific contexts, disciplines, or regions. The title claims to address 'higher education' globally but draws on a narrow set of examples." 125 }, 126 "alternative_explanations_discussed": { 127 "applies": false, 128 "answer": false, 129 "justification": "This is a theoretical/position paper presenting no empirical results of its own." 130 }, 131 "proxy_outcome_distinction": { 132 "applies": false, 133 "answer": false, 134 "justification": "No measurements taken; theoretical paper." 135 } 136 }, 137 "setup_transparency": { 138 "model_versions_specified": { 139 "applies": false, 140 "answer": false, 141 "justification": "No models used in experiments." 142 }, 143 "prompts_provided": { 144 "applies": false, 145 "answer": false, 146 "justification": "No prompting used." 147 }, 148 "hyperparameters_reported": { 149 "applies": false, 150 "answer": false, 151 "justification": "No experiments conducted." 152 }, 153 "scaffolding_described": { 154 "applies": false, 155 "answer": false, 156 "justification": "No agentic scaffolding used." 157 }, 158 "data_preprocessing_documented": { 159 "applies": false, 160 "answer": false, 161 "justification": "No data collected or processed." 162 } 163 }, 164 "limitations_and_scope": { 165 "limitations_section_present": { 166 "applies": true, 167 "answer": false, 168 "justification": "No limitations section or discussion of the paper's own limitations anywhere in the text." 169 }, 170 "threats_to_validity_specific": { 171 "applies": true, 172 "answer": false, 173 "justification": "No threats to validity discussed." 174 }, 175 "scope_boundaries_stated": { 176 "applies": true, 177 "answer": false, 178 "justification": "No explicit scope boundaries stated. The paper addresses 'higher education' broadly without acknowledging what it does not cover." 179 } 180 }, 181 "data_integrity": { 182 "raw_data_available": { 183 "applies": false, 184 "answer": false, 185 "justification": "No data collected; theoretical paper." 186 }, 187 "data_collection_described": { 188 "applies": false, 189 "answer": false, 190 "justification": "No data collection; this is a commentary/position paper." 191 }, 192 "recruitment_methods_described": { 193 "applies": false, 194 "answer": false, 195 "justification": "No participants or data sources." 196 }, 197 "data_pipeline_documented": { 198 "applies": false, 199 "answer": false, 200 "justification": "No data pipeline." 201 } 202 }, 203 "conflicts_of_interest": { 204 "funding_disclosed": { 205 "applies": true, 206 "answer": false, 207 "justification": "No funding information or acknowledgments section present in the paper." 208 }, 209 "affiliations_disclosed": { 210 "applies": true, 211 "answer": true, 212 "justification": "Author affiliation is listed: Richmond College, UK." 213 }, 214 "funder_independent_of_outcome": { 215 "applies": true, 216 "answer": false, 217 "justification": "No funding information disclosed, so independence cannot be assessed." 218 }, 219 "financial_interests_declared": { 220 "applies": true, 221 "answer": false, 222 "justification": "No competing interests statement present in the paper." 223 } 224 }, 225 "contamination": { 226 "training_cutoff_stated": { 227 "applies": false, 228 "answer": false, 229 "justification": "Paper does not evaluate any pre-trained model on benchmarks." 230 }, 231 "train_test_overlap_discussed": { 232 "applies": false, 233 "answer": false, 234 "justification": "No model evaluation on benchmarks." 235 }, 236 "benchmark_contamination_addressed": { 237 "applies": false, 238 "answer": false, 239 "justification": "No benchmark evaluation." 240 } 241 }, 242 "human_studies": { 243 "pre_registered": { 244 "applies": false, 245 "answer": false, 246 "justification": "No human participants in this paper." 247 }, 248 "irb_or_ethics_approval": { 249 "applies": false, 250 "answer": false, 251 "justification": "No human participants." 252 }, 253 "demographics_reported": { 254 "applies": false, 255 "answer": false, 256 "justification": "No human participants." 257 }, 258 "inclusion_exclusion_criteria": { 259 "applies": false, 260 "answer": false, 261 "justification": "No human participants." 262 }, 263 "randomization_described": { 264 "applies": false, 265 "answer": false, 266 "justification": "No human participants." 267 }, 268 "blinding_described": { 269 "applies": false, 270 "answer": false, 271 "justification": "No human participants." 272 }, 273 "attrition_reported": { 274 "applies": false, 275 "answer": false, 276 "justification": "No human participants." 277 } 278 }, 279 "cost_and_practicality": { 280 "inference_cost_reported": { 281 "applies": false, 282 "answer": false, 283 "justification": "Theoretical paper; no method with inference costs." 284 }, 285 "compute_budget_stated": { 286 "applies": false, 287 "answer": false, 288 "justification": "No computation performed." 289 } 290 } 291 }, 292 "claims": [ 293 { 294 "claim": "Over-reliance on AI tools by students threatens development of critical thinking and synthesis skills.", 295 "evidence": "Cites Dwivedi et al. (2023) for negative correlation between frequent AI use and critical-thinking performance, and Carr (2020) on passive information consumption. No original empirical evidence provided.", 296 "supported": "weak" 297 }, 298 { 299 "claim": "Experienced developers using AI actually took 19% longer to finish tasks.", 300 "evidence": "Cites Klang & Fuller (2025) from METR.org. This is a secondary citation — the paper does not conduct this study.", 301 "supported": "moderate" 302 }, 303 { 304 "claim": "AI-generated text evades traditional plagiarism detection because it is synthetically original.", 305 "evidence": "Cites Khalil & Er (2023). Reasonable claim supported by the cited reference.", 306 "supported": "moderate" 307 }, 308 { 309 "claim": "Using AI to study didn't greatly change medical students' test scores.", 310 "evidence": "Cites Bharat & Smith (2025). Secondary citation, not original research.", 311 "supported": "moderate" 312 } 313 ], 314 "red_flags": [ 315 { 316 "flag": "No original empirical contribution", 317 "detail": "The paper presents no original data, experiments, or systematic review. All empirical claims are secondary citations from other papers, making this a commentary/opinion piece rather than a research paper." 318 }, 319 { 320 "flag": "Claims significantly outrun the evidence", 321 "detail": "Sweeping claims about AI's impact on all of higher education are made based on a handful of selectively cited studies. No systematic literature search or structured review methodology is employed." 322 }, 323 { 324 "flag": "No limitations discussed", 325 "detail": "The paper makes broad prescriptive claims about restructuring higher education without acknowledging any limitations, uncertainties, or boundary conditions." 326 }, 327 { 328 "flag": "Selective citation", 329 "detail": "References are cherry-picked to support the narrative. For example, the Klang & Fuller (2025) finding about developers being 19% slower is presented without noting it was specific to experienced open-source developers on a particular task type." 330 } 331 ], 332 "cited_papers": [ 333 { 334 "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity", 335 "authors": ["E. Klang", "R. Fuller"], 336 "year": 2025, 337 "relevance": "RCT measuring AI coding tool impact on experienced developer productivity — directly relevant to the survey's scope on AI programming productivity." 338 }, 339 { 340 "title": "Will ChatGPT get you caught? Rethinking of plagiarism detection", 341 "authors": ["M. Khalil", "E. Er"], 342 "year": 2023, 343 "arxiv_id": "2302.04335", 344 "relevance": "Examines whether AI-generated text can be detected by plagiarism tools, relevant to AI code/text generation detection." 345 }, 346 { 347 "title": "\"So what if ChatGPT wrote it?\" Multidisciplinary perspectives on opportunities, challenges and implications of generative conversational AI for research, practice and policy", 348 "authors": ["Y. K. Dwivedi", "N. Kshetri", "L. Hughes"], 349 "year": 2023, 350 "doi": "10.1016/j.ijinfomgt.2023.102642", 351 "relevance": "Broad multidisciplinary assessment of generative AI impacts including on research practices." 352 }, 353 { 354 "title": "Systematic review of research on artificial intelligence applications in higher education – where are the educators?", 355 "authors": ["O. Zawacki-Richter", "V. I. Marín", "M. Bond", "F. Gouverneur"], 356 "year": 2019, 357 "doi": "10.1186/s41239-019-0171-0", 358 "relevance": "Systematic review of AI in higher education — relevant as a methodological comparison for survey quality." 359 } 360 ] 361 }