ai-research-survey

scan.json (30168B)

---

 {
   "paper": {
     "title": "A GPT-based Code Review System for Programming Language Learning",
     "authors": ["Lee Dong-Kyu"],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2407.04722",
     "doi": "10.48550/arXiv.2407.04722"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study", "qualitative"],
   "key_findings": "The paper presents a GPT-4-based code review system for programming education that catches error types (hard coding at 21.3%, unnecessary code at 17.59%) missed by traditional online judge systems. The improved system reduces response time by 12-58% and API costs by up to 8.53% compared to the initial version through prompt optimization and a code validation module. A survey of 6 software education experts rated code review quality positively (mostly 4-5 on 5-point Likert), but the system was never tested with its target audience of primary and secondary school students.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a deployed URL (https://www.codetutor119judge.com/) but no source code repository. A live web app is not source code release."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The 93-item test dataset collected from Company C's Online Judge System is described but not publicly released. No download link or repository is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Next.js, Monaco Editor, and Azure Static Web App but provides no dependency versions, requirements.txt, or environment specification sufficient to recreate the setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The paper describes the system architecture but does not provide steps to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (failure rates in Table 3, response times in Table 4, costs in Table 5, survey in Table 6) are reported as point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the improved system is better across all metrics (correctness, response time, cost) but performs no statistical significance tests. All comparisons are raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Response time reductions are reported with baseline context (e.g., 12-58% reduction with both initial and improved values in Table 4). Cost reduction reported as 8.53% with per-call costs for both systems. Error type failure rates show absolute percentages."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the sample sizes: 108 test codes for correctness evaluation, 92 error codes for response time, 6 evaluators for quality survey. No power analysis or acknowledgment that these may be too small."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experiment. Response times appear to be single measurements per exercise, not averaged across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The improved system is compared against two baselines: the existing online judge system (for error detection, RQ1) and the initial version of the system (for response time and cost, RQ2-3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baseline is a traditional online judge system using test-case matching. No comparison to other LLM-based code review or feedback systems (e.g., CodeAid, which is cited in the paper) is included."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "Multiple changes were made between the initial and improved system (code validation module added, maxTokens optimized, temperature/topP adjusted, prompt restructured). No ablation isolates the contribution of individual changes."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four distinct evaluation dimensions are used: error type detection rates (RQ1), response time (RQ2), API cost (RQ3), and expert-rated quality on 5 criteria (RQ4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Six software education experts evaluated code review quality using a 5-point Likert scale across 5 criteria (precision, usefulness, specificity, supportive tone, learning effect). Results in Table 6."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The prompts were developed and iteratively improved using data from the same online judge system. It is unclear whether the 108 evaluation test codes overlap with the 93 development test data. No explicit dev/test separation is described."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Error detection is broken down by 4 error types (unnecessary code, requirement not met, hard coding, computation error) in Table 3. Survey results are broken down by 5 quality criteria in Table 6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or error analysis is discussed. The paper reports aggregate success rates but does not examine specific cases where the system failed or produced poor reviews."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows improvement across all metrics. No negative results, failed approaches, or configurations that hurt performance are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about identifying error types, shortening response times, lowering API costs, and maintaining quality are each addressed by RQ1-4 in the results sections with corresponding tables and figures."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implicitly claims that prompt optimization and the new code validation module caused the improvements. However, multiple changes were made simultaneously (prompt restructuring, maxTokens optimization, temperature/topP adjustment, validation module) without controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims applicability to 'Programming Language Learning' broadly. The abstract says 'suitability for teaching programming to primary and secondary school students.' However, the system was only tested with adult experts, only on Python, and only on exercises from one online judge system."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered for the results. For instance, response time improvements could be due to API-side changes, and error detection improvements could reflect GPT-4's existing knowledge of the exercises rather than the prompt design."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures expert Likert ratings of code review quality and claims the system is suitable for student programming education. The gap between expert ratings (proxy) and actual student learning outcomes (claimed outcome) is never acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-4' throughout without specifying a version (e.g., gpt-4-0613) or snapshot date. Model behavior changes across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figures 3 and 5 show the module structure with role-setting, RNP, and RCG prompt components, but these are templates with placeholders ('Exercise', 'Submitted Code', 'Solution'). The actual prompt text appears to be in Korean in figures, and the paper describes prompt content in natural language rather than providing verbatim reproducible prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'the maxTokens value was optimized' and 'the values of the temperature and topP parameters were adjusted' but never reports the actual values used for any of these parameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The system is a sequential pipeline (code validation → correctness check → code review) using single GPT-4 API calls, not an agentic scaffold with tool use, loops, or autonomous decision-making."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III.A documents the data collection pipeline: raw data from Company C's Online Judge System starting 2021, static-based filtering to remove repeated submissions and comments, resulting in 93 test data for 27 questions."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Section V ('Conclusion and Further Study') contains one sentence about future work ('further improve usability, introduce a membership management system, and verify the effectiveness with primary and secondary school students') but no substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. Issues such as the tiny evaluator sample, lack of student participants, prompt-data overlap, and GPT-4 version sensitivity are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do not show. It mentions future work (testing with students) but does not bound its claims by stating what was not tested or what populations/settings are excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the test dataset nor the survey responses are made publicly available. No supplementary data files or download links are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III.A describes data collection: exercises, student-submitted codes, and instructor answer codes from Company C's Online Judge System starting in 2021, with static-based filtering to remove repeated submissions and delete comments."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states participants were 'software education experts' with 'at least two years of programming education experience' but does not describe how they were recruited (convenience sampling, institutional affiliation, solicitation method)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The initial data pipeline (raw collection → static filtering → 93 test data) is described in Section III.A. However, the evaluation uses 108 test codes and 92 error codes with no explanation of how these differ from the 93 development test data or how they were derived."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author's affiliation is listed: Department of Electrical Engineering and Computer Science, University of Hanyang, Seoul, Republic of Korea."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of the funder cannot be assessed. The system uses Company C's Online Judge System data, but the relationship between the author and Company C is not clarified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4 to evaluate code correctness and generate reviews but never states GPT-4's training data cutoff date. The online judge exercises could be in GPT-4's training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the programming exercises from Company C's Online Judge System could appear in GPT-4's training data. This is a significant concern since GPT-4 is used as the code correctness checker."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The online judge exercises have been available since 2021 and could have been in GPT-4's training data. The paper does not address this contamination risk despite using GPT-4 to check code correctness against these same exercises."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned. The study involves human participants (3 usability testers, 6 survey evaluators) but no pre-registration link is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite involving human participants in usability testing and survey evaluation."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Participants are described as 'software education experts' with 'at least two years of programming education experience.' No other demographics (gender, age, geographic distribution, specific programming expertise) are reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "An inclusion criterion is stated: participants were 'current instructors with more than 2 years of programming education experience.' The survey also specifies participant roles (3 prior participants, 2 additional instructors, 1 coding content developer)."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study with treatment/control conditions. It is a usability evaluation and survey, so randomization is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a usability evaluation and survey of a single system, not a comparative experimental study where blinding would be applicable."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition information is reported. All 6 survey participants appear to have completed the evaluation, but this is not explicitly stated."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 specifically measures API call costs. Table 5 reports input/output tokens and per-call costs in USD for both the initial and improved systems, showing up to 8.53% cost reduction."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Per-call costs are reported but no total computational budget (total API spend across all experiments, total number of API calls, Azure hosting costs) is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multiple runs or seed variation is reported. GPT-4 outputs can vary across calls (especially with non-zero temperature), but results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many times each experiment was run. Response times and code correctness results appear to be single measurements."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper states temperature, topP, and maxTokens were 'optimized' and 'adjusted' but reports neither the final values nor the search budget (how many configurations were tried)."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports five iterations of prompt improvement but does not describe the selection criterion for the final configuration or whether selection was done on held-out data."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system without acknowledging self-evaluation bias. The expert evaluators may have known the system's goals, and no independent evaluation was conducted."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "The system is a single GPT-4 API-based tool. Compute budget is the subject of RQ3 (cost reduction), not a confound between compared methods."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 108 test codes from one online judge system are representative of the errors primary/secondary students would actually make, or whether the 5-point Likert expert survey measures what makes a code review effective for young learners."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The system itself is the product being evaluated, not a model comparison confounded by scaffolding differences."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The online judge exercises have been available since 2021. GPT-4's training data likely includes similar programming exercises. This temporal overlap is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The code correctness module provides GPT-4 with the instructor's solution code alongside the student submission. This gives the model the answer key, but the paper does not discuss how this affects the evaluation of GPT-4's code review ability."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 108 test codes come from the same 27 exercises, meaning multiple test cases per exercise share the same problem structure. Independence of test samples is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied. No canary strings, membership inference, or contamination analysis is performed."
       }
     }
   },
   "claims": [
     {
       "claim": "The improved system identifies error types (hard coding, unnecessary code, requirement not met, computation error) more effectively than the existing online judge system.",
       "evidence": "Table 3 and Figure 11 show failure rates by error type: hard coding 21.3%, unnecessary code 17.59%, requirement not met 11.11%, computation error 6.48%. The existing judge system passed these codes; the improved system caught them.",
       "supported": "moderate"
     },
     {
       "claim": "The improved system reduces response time for code review comments by 12-58% compared to the initial system.",
       "evidence": "Table 4 and Figure 12 show response time comparisons across 27 exercises between the initial and improved systems, with consistent reductions.",
       "supported": "moderate"
     },
     {
       "claim": "The improved system reduces API call costs by up to 8.53% compared to the initial system.",
       "evidence": "Table 5 and Figure 13 show average token counts and per-call costs in USD for both systems, with the improved system using fewer tokens through prompt optimization.",
       "supported": "moderate"
     },
     {
       "claim": "The system maintains high-quality code reviews despite reduced response times and API costs.",
       "evidence": "Table 6 shows Likert-scale survey results from 6 experts across 5 criteria (precision, usefulness, specificity, supportive tone, learning effect), with mostly positive ratings.",
       "supported": "weak"
     },
     {
       "claim": "The system is suitable for teaching programming to primary and secondary school students.",
       "evidence": "Expert survey feedback states suitability, but no testing with actual students was conducted. Only adult software education experts evaluated the system.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No testing with target audience",
       "detail": "The system is designed for primary and secondary school students but was evaluated exclusively by adult software education experts. The paper's central claim of suitability for young learners has no direct evidence from the target population."
     },
     {
       "flag": "Tiny evaluator sample",
       "detail": "Only 3 experts participated in the usability test and 6 in the quality survey. This is too small to draw reliable conclusions about system quality, and no statistical significance can be established."
     },
     {
       "flag": "GPT-4 used as code correctness checker without validation",
       "detail": "Using GPT-4 to determine if code is correct (instead of actually compiling and running it) is unreliable. The paper uses GPT-4 to 'function as the compiler' (Figure 5) without discussing the error rate of this approach or comparing it to actual compilation."
     },
     {
       "flag": "Potential dev/test overlap",
       "detail": "The same online judge system exercises were used for both prompt development (93 test data) and evaluation (108 test codes). No clear separation between development and test data is documented."
     },
     {
       "flag": "No statistical tests on any claims",
       "detail": "All claims of improvement (error detection, response time, cost) are based on comparing raw numbers without significance tests, despite variability in GPT-4 outputs being well-known."
     },
     {
       "flag": "Hyperparameter values withheld",
       "detail": "Temperature, topP, and maxTokens were 'optimized' but the actual values are never reported, making the experiments unreproducible."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "GPT-4 may have seen similar or identical programming exercises in its training data. The paper uses GPT-4 as the code correctness checker, so contamination directly affects the core evaluation. This risk is never discussed despite being acknowledged in the related work (Section II.D on data leakage)."
     }
   ],
   "cited_papers": [
     {
       "title": "CodeAid: Evaluating a Classroom Deployment of an LLM-based Programming Assistant that Balances Student and Educator Needs",
       "authors": ["Majeed Kazemitabaar", "Runlong Yang", "Xiaoning Wang", "Austin Zachary Henley", "Paul Denny", "Michelle Craig", "Tovi Grossman"],
       "year": 2024,
       "doi": "10.1145/3613904.3642773",
       "relevance": "Directly relevant as an LLM-based programming assistant deployment study that balances learning with AI assistance, addressing the same anti-cheating concern."
     },
     {
       "title": "Natural Language Generation and Understanding of Big Code for AI-Assisted Programming: A Review",
       "authors": ["Man-Fai Wong", "Shangxin Guo", "Ching-Nam Hang", "Siu-Wai Ho", "Chee-Wei Tan"],
       "year": 2023,
       "doi": "10.3390/e25060888",
       "relevance": "Survey of LLM capabilities for AI-assisted programming, covering code generation and understanding relevant to automated code review."
     },
     {
       "title": "Can Large Language Models Provide Feedback to Students? A Case Study on ChatGPT",
       "authors": ["Wei Dai", "Flora Jin", "Tongguang Li", "Yi-Shan Tsai"],
       "year": 2023,
       "relevance": "Evaluates ChatGPT's ability to generate educational feedback on student assignments, directly relevant to LLM-based feedback quality."
     },
     {
       "title": "Automated feedback generation for introductory programming assignments",
       "authors": ["Rishabh Singh", "Sumit Gulwani", "Armando Solar-Lezama"],
       "year": 2013,
       "doi": "10.1145/2491956.2462195",
       "relevance": "Foundational work on automated programming feedback that proposed minimal-correction feedback, an alternative to LLM-based approaches."
     },
     {
       "title": "LLaMA-Reviewer: Advancing Code Review Automation with Large Language Models through Parameter-Efficient Fine-Tuning",
       "authors": ["Junyi Lu", "Lei Yu", "Xiaojia Li", "Li Yang", "Chun Zuo"],
       "year": 2023,
       "relevance": "Proposes an LLM-based code review automation pipeline with systematic prompt templates, directly relevant to automated code review methodology."
     },
     {
       "title": "A Critical Review of Large Language Model on Software Engineering: An Example from ChatGPT and Automated Program Repair",
       "authors": ["Quanjun Zhang"],
       "year": 2023,
       "relevance": "Evaluates ChatGPT's automated program repair capability on a contamination-controlled dataset, relevant to LLM code fixing and data leakage concerns."
     },
     {
       "title": "From 'Ban It Till We Understand It' to 'Resistance is Futile': How University Programming Instructors Plan to Adapt as More Students Use AI Code Generation and Explanation Tools such as ChatGPT and GitHub Copilot",
       "authors": ["Sam Lau", "Philip J. Guo"],
       "year": 2023,
       "relevance": "Studies instructor attitudes toward LLM tools in programming education, including concerns about AI-assist cheating."
     },
     {
       "title": "Designing LLM-based Code Reviewing Learning Environment for Programming Education",
       "authors": ["Seong-yune Choi", "Donghee Lee", "Jungho Kim", "Youngkwang Jang", "Hyungshin Kim"],
       "year": 2023,
       "doi": "10.32431/kace.2023.26.5.001",
       "relevance": "Proposes LLM-based code review for programming education with verification criteria used as reference for the quality survey in this paper."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "A deployed web tool for programming education with a working URL, but limited to one online judge's exercises and Korean-language primary/secondary students."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms expected results that GPT-4 can generate code reviews and reduce instructor workload; no surprising or contrarian findings."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the paper addresses AI-assist cheating as a design constraint, not as a demonstrated risk."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenge to existing work, and no debate-worthy findings."
     },
     "demo_ability": {
       "score": 2,
       "justification": "A live web application at codetutor119judge.com is mentioned, though it targets Korean-language exercises and no source code is available."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses GPT-4 (recognized brand) but the author and institution are not widely known in the AI/SE research community."
     }
   }
 }