ai-research-survey

scan.json (20679B)

---

 {
   "paper": {
     "title": "Automated Knowledge Component Generation for Interpretable Knowledge Tracing in Coding Problems",
     "authors": ["Zhangqi Duan", "Nigel Fernandez", "Arun Balajiee Lekshmi Narayanan", "Mohammad Hassany", "Rafaella Sampaio de Alencar", "Peter Brusilovsky", "Bita Akram", "Andrew Lan"],
     "year": 2025,
     "venue": "Preprint (under review)",
     "arxiv_id": "2502.18632"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We will make our code publicly available' (Section 1.1 footnote), which is a promise of future release, not an actual release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two publicly available datasets: CodeWorkout (DataShop, pslcdatashop.web.cmu.edu) and FalconCode, both with citations and download links."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using HuggingFace PEFT library, Llama 3 8B, NVIDIA L40S 48GB GPU, and LoRA hyperparameters, but no requirements.txt, Dockerfile, or detailed environment specification is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The methodology is described but not in a form a researcher could directly execute."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Standard deviations are reported (e.g., '0.816±0.012' in Table 2) but no confidence intervals are provided. The ± notation represents standard deviation across 5 splits, not confidence intervals."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 notes statistical significance with p < 0.05, indicated by bullet markers (•) for KCGen-KT results compared to baselines."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context enabling effect size interpretation, e.g., AUC improvement from 0.788 (TIKTOC*) to 0.816 (KCGen-KT) on CodeWorkout, with absolute differences interpretable from Table 2."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The datasets contain 246 students (CodeWorkout) and 3,267 students (FalconCode) but no justification for why these sizes are adequate for the claims, nor any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations across 5 repeated random subsampling splits are reported in Table 2 (e.g., '0.816±0.012')."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: Random, Majority, Code-DKT, TIKTOC*, and KCGen-KT with human-written KCs (Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "TIKTOC (2025) and Code-DKT (2022) are recent and relevant baselines for knowledge tracing in programming."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Tables 4 and 5 present detailed ablation studies examining the impact of correct solutions, in-context examples, AST vs code input, number of solutions, and KC abstraction levels."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are used: AUC, F1 Score, Accuracy for correctness prediction, and CodeBLEU for code prediction (Table 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 presents a human evaluation with two annotators assessing KC interpretability (98.6% labeled interpretable), precision (93.2%), and recall (96% equal-or-greater coverage) with inter-annotator agreement reported."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states 'repeated random subsampling with five splits, partitioning the data into training, validation, and test sets each time.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (CodeWorkout and FalconCode) in Table 2, and per KC abstraction level in Table 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2.1 shows a case study where predicted code has a runtime error due to low mastery on specific KCs. Section 5.2.2 discusses learning curves that poorly fit due to noisy data."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study reports that incorporating incorrect solutions hurts performance, and using AST representations decreases performance (Section 5.1.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims KCGen-KT outperforms existing KT methods and human-written KCs, which is supported by Table 2 with statistical significance. The claim about better learning curves is supported by R² comparison (0.21 vs 0.18)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'leveraging the semantic information in KC descriptions improves KT performance' are supported by controlled ablation studies (Tables 4-5) that isolate individual components."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper tests on two specific datasets (Java and Python) and discusses limitations in Section 7, including plans to extend to other domains. The title specifies 'coding problems' rather than making broader claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for why LLM-generated KCs outperform human KCs. For example, it does not consider whether the improvement could be due to differences in KC granularity rather than quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o' for KC generation and 'Llama 3' with 8B parameters for KT, but no specific version identifiers or snapshot dates are provided (e.g., no gpt-4o-2024-xx-xx or Llama-3-8B-Instruct version)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided in Appendix B (Tables 7-10) for KC generation, cluster summarization, KC correctness labeling, and in-context example conversion."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 reports LoRA parameters (α=256, rank=128, dropout=0.05), learning rates for each component, batch size (32), optimizer choices (AdamW, RMSprop), and 8-bit quantization."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-step KC generation pipeline is described in detail (Section 3.1): initial KC generation with few-shot prompting, clustering on semantic similarity, and cluster labeling with GPT-4o."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes filtering to first submissions only (10,834 from CodeWorkout, 28,617 from FalconCode), and Section 3.1.1 describes the clustering of student submissions using CodeBERT embeddings for diverse sampling."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Some limitations are briefly mentioned in Section 7 (Conclusions and Future Work) but without substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The future work section mentions extending to other domains but does not address specific validity concerns."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do not show. It does not discuss limitations of testing on only introductory programming courses or the two specific datasets."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The underlying datasets (CodeWorkout and FalconCode) are publicly available through their respective repositories, allowing independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes both datasets: CodeWorkout from an introductory Java course (246 students, 50 problems), FalconCode from an introductory Python course (3,267 students, 157 problems), with citation to original data sources."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses existing public datasets of student submissions, not recruiting participants. The human evaluation annotators are described as 'volunteers contacted through a research partner' (Appendix A)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw student submissions through CodeBERT embedding, clustering, KC generation, KC clustering, and final tagging is documented step by step in Section 3.1, with specific counts at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Massachusetts Amherst, University of Pittsburgh, and North Carolina State University. No authors are affiliated with OpenAI or Meta (whose products are used)."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed; appears to be unfunded academic work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o is used for KC generation and Llama 3 for KT, but neither model's training data cutoff is stated. The student code datasets could potentially be in GPT-4o's training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4o may have seen the CodeWorkout or FalconCode datasets during training, which could affect KC generation quality."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CodeWorkout (2019/2021) and FalconCode (2023) are publicly available datasets that could be in GPT-4o's training data. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation in Section 6 involves two human annotators but no pre-registration is mentioned."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics approval is mentioned for the human evaluation study involving annotators."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The annotators are described only as having 'experience in teaching college-level programming in Java.' No further demographics are reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion/exclusion criteria for annotator selection are stated beyond Java teaching experience."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study with treatment/control groups for the human evaluation; both annotators evaluated the same materials."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "For the recall task, annotators compare two KC sets but it is not stated whether they were blinded to which set was LLM-generated vs. human-written."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "With only two annotators, attrition is less relevant, but no mention is made of whether both completed all annotation tasks."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs for GPT-4o calls or inference costs for the KC generation pipeline are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states training takes 80 minutes per epoch on CodeWorkout and 300 minutes per epoch on FalconCode on an NVIDIA L40S 48GB GPU, converging within 12 epochs."
       }
     }
   },
   "claims": [
     {
       "claim": "KCGen-KT with LLM-generated KCs outperforms existing KT methods and human-written KCs on future student response prediction",
       "evidence": "Table 2 shows statistically significant improvements (p<0.05) on both datasets across all metrics. On CodeWorkout: AUC 0.816 vs 0.788 (TIKTOC*) and 0.797 (human KCs). On FalconCode: AUC 0.771 vs 0.728 (TIKTOC*) and 0.752 (human KCs).",
       "supported": "strong"
     },
     {
       "claim": "LLM-generated KCs result in better fit than human-written KCs under cognitive models (power law of practice)",
       "evidence": "Section 5.2.2 reports weighted R² of 0.21 for LLM-generated KCs vs 0.18 for human-written KCs under PFA models on CodeWorkout.",
       "supported": "moderate"
     },
     {
       "claim": "LLM-generated KCs are comparable to or more interpretable than human-written KCs",
       "evidence": "Section 6 reports 98.6% of LLM-generated KCs labeled interpretable vs 94.6% for baseline, precision 93.2% vs 92.5%, and recall showing equal-or-greater coverage in 96% of cases. Cohen's Kappa of 0.594 for inter-annotator agreement.",
       "supported": "moderate"
     },
     {
       "claim": "Including correct student submissions is crucial for KC generation quality",
       "evidence": "Table 4 ablation: removing correct solutions drops AUC from 0.812 to 0.789 and CodeBLEU from 0.569 to 0.529.",
       "supported": "strong"
     },
     {
       "claim": "Overly abstract KCs hurt KT performance",
       "evidence": "Table 3 shows consistent performance decrease at highest abstraction levels (e.g., 10 clusters: AUC 0.794 vs 50 clusters: AUC 0.816 on CodeWorkout).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper presents KCGen-KT, an LLM-based pipeline for automatically generating knowledge components (KCs) for programming problems and using them for knowledge tracing. LLM-generated KCs outperform human-written KCs on student response prediction across two datasets (CodeWorkout/Java and FalconCode/Python) with statistical significance. Ablation studies show that including diverse correct student solutions and maintaining medium-level KC granularity are important for performance. A human evaluation with two annotators confirms that LLM-generated KCs are interpretable and achieve comparable precision/recall to human-authored KCs.",
   "red_flags": [
     {
       "flag": "Tiny human evaluation sample",
       "detail": "The human evaluation uses only 2 annotators, which is insufficient for robust inter-rater reliability assessment. Cohen's Kappa of 0.594 (moderate agreement) on KC interpretability suggests the task is subjective, yet no additional annotators were recruited."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section. Key threats such as dataset contamination risk for GPT-4o and generalizability beyond introductory programming courses are not discussed."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "GPT-4o is used for KC generation on publicly available datasets (CodeWorkout 2019, FalconCode 2023). If GPT-4o was trained on these datasets, the quality of generated KCs may be inflated. This is not discussed."
     },
     {
       "flag": "Code not released",
       "detail": "Despite promising code release, no repository link is provided, limiting reproducibility."
     }
   ],
   "cited_papers": [
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "relevance": "Pre-trained code model used for embedding student submissions in the KC generation pipeline."
     },
     {
       "title": "Code-DKT: A Code-based Knowledge Tracing Model for Programming Tasks",
       "authors": ["Yang Shi", "Min Chi", "Tiffany Barnes", "Thomas Price"],
       "year": 2022,
       "relevance": "Key baseline for knowledge tracing in programming that leverages student code content."
     },
     {
       "title": "Open-ended Knowledge Tracing for Computer Science Education",
       "authors": ["Naiming Liu", "Zichao Wang", "Richard Baraniuk", "Andrew Lan"],
       "year": 2022,
       "relevance": "Prior LLM-based knowledge tracing method for programming that predicts student code submissions."
     },
     {
       "title": "Test Case-Informed Knowledge Tracing for Open-ended Coding Tasks",
       "authors": ["Zhangqi Duan", "Nigel Fernandez", "Alexander Hicks", "Andrew Lan"],
       "year": 2025,
       "relevance": "Main baseline (TIKTOC) using LLM backbone for multi-task knowledge tracing in programming."
     },
     {
       "title": "The Llama 3 Herd of Models",
       "authors": ["AI @ Meta Llama Team"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Open-source LLM used as the backbone for the KCGen-KT knowledge tracing framework."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["Edward J Hu"],
       "year": 2022,
       "relevance": "Parameter-efficient fine-tuning method used for adapting Llama 3 in the KT framework."
     },
     {
       "title": "Automated generation and tagging of knowledge components from multiple-choice questions",
       "authors": ["Steven Moore", "Robin Schmucker", "Tom Mitchell", "John Stamper"],
       "year": 2024,
       "relevance": "Prior work on LLM-based KC generation in science domain, related approach for different problem types."
     },
     {
       "title": "Harnessing code domain insights: Enhancing programming Knowledge Tracing with Large Language Models",
       "authors": ["Xinjie Sun", "Qi Liu", "Kai Zhang"],
       "year": 2025,
       "relevance": "Concurrent work using LLMs to construct Q-matrices for knowledge tracing in programming."
     }
   ]
 }