ai-research-survey

scan.json (29796B)

---

 {
   "paper": {
     "title": "Rubric Is All You Need: Improving LLM-based Code Evaluation With Question-Specific Rubrics",
     "authors": [
       "Aditya Pathak",
       "Rachit Gandhi",
       "Vaibhav Uttam",
       "Arnav Ramamoorthy",
       "Pratyush Ghosh",
       "Aaryan Raj Jindal",
       "Shreyash Verma",
       "Aditya Mittal",
       "Aashna Ased",
       "Chirag Khatri",
       "Yashwanth Nakka",
       "Devansh",
       "Jagat Sesh Challa",
       "Dhruv Kumar"
     ],
     "year": 2025,
     "venue": "International Computing Education Research Workshop",
     "arxiv_id": "2503.23989",
     "doi": "10.1145/3702652.3744220"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Question-specific rubrics substantially outperform question-agnostic rubrics for LLM-based code grading, especially on algorithmically diverse DSA problems (ICC3 jumps from 0.560 to 0.819). The Complete Rubric Evaluation (CRE) technique achieves near-human leniency (0.082), while Pointwise Rubric Evaluation (PRE) is significantly harsher (leniency -0.329) due to single-criterion prompts forcing zero-or-full marking. LLM-based techniques vastly outperform similarity-based approaches like CodeBERTScore. The study introduces two datasets (80 OOP, 150 DSA submissions) and a new Leniency metric.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided: https://github.com/BITS-Pilani-GRC/Rubric-Grader (Section 1, footnote 2)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset released on HuggingFace: https://huggingface.co/datasets/BITS-Pilani-GRC/RubricEval (Section 3, footnote 1/3)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification mentioned. The paper only names the models used (GPT-4o-mini, Claude 3.7 Sonnet) without dependency or version details for the evaluation code."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. A GitHub link is given but the paper itself contains no 'Reproducing Results' section or specific commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2 and 3 report only point estimates for all metrics (Pearson r, Spearman rs, ICC, Cohen's Kappa, Leniency). No confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'question-specific rubrics significantly enhance logical assessment') but no statistical significance tests (t-tests, bootstrap, etc.) are applied to any comparison. All conclusions are drawn from visual inspection of metric values."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports correlation values and absolute metric changes with context, e.g., 'lifts ICC3 from 0.560 → 0.819 and boosts Pearson r by +0.26 points (0.562 → 0.825)' in Section 6.2.2. All metrics in Tables 2 and 3 provide magnitude information."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 80 OOP submissions and 150 DSA submissions were chosen. No power analysis. The selection of 20 per score range for OOP is described as a sampling procedure but the overall N is not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation reported across experimental runs. LLM outputs are non-deterministic but all results appear to be from single runs. No mention of repeated evaluations or spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines compared: CodeBERTScore (no LLM), CodeJudge (no rubric), Five Point Marking (question-agnostic), EME-QA (question-agnostic). See Tables 2 and 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are recent: CodeBERTScore (2023), CodeJudge (2024), CodEV (2024), Phung et al. FPM (2023). All from the last 2 years relative to the paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper effectively ablates components: QA vs QS rubrics within EME, CRE vs PRE (whole-rubric vs pointwise evaluation), varying ensemble sizes (Section 6.2.4), and model comparisons (CRE OpenAI vs CRE Claude in Table 2)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Seven complementary metrics used across three dimensions: correlations (Pearson, Spearman, Kendall-Tau), agreement (ICC1/2/3, Cohen's Kappa), and bias (Leniency). Described in Section 5.3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Ground truth consists of human grades by TAs and final-year students using consensus-driven approach. The entire study evaluates automated grading against human expert assessment."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No separation of development and test data. All 80 OOP and 150 DSA submissions are used for evaluation. Prompts (Appendix A) were presumably developed and iterated on the same data used for final evaluation, but no dev/test split is documented."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by dataset (OOP Table 2, DSA Table 3), by rubric type (QA vs QS), by technique (CRE, PRE, EME, FPM, CodeJudge, CodeBERTScore), and by model (OpenAI vs Claude for CRE)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2.3 discusses PRE's excessive strictness (leniency -0.329) and explains why single-criterion prompts force harsh grading. Section 6.2.2 discusses where QA rubrics fail on diverse DSA problems. Section 7.1 discusses when each technique may be inappropriate."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "PRE's poor leniency (-0.329) is a negative result. CRE with Claude underperforms CRE with OpenAI on OOP (Pearson 0.840 vs 0.912). FPM's poor Kappa (0.346 OOP, 0.072 DSA) is reported without downplaying."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that question-specific rubrics 'significantly enhance logical assessment' are supported by Tables 2 and 3, particularly the large DSA improvements (EME-QS ICC3 0.819 vs EME-QA 0.560). The claim about the new Leniency metric is supported by its formal definition and application."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims question-specific rubrics improve evaluation quality. The study design compares the same technique (EME) with QA vs QS rubrics on the same data, isolating the rubric type as the manipulated variable. This controlled comparison is adequate for the causal claim within the tested setting."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Rubric Is All You Need' implies universal applicability. The abstract claims 'question-specific rubrics significantly enhance logical assessment of code in educational settings' broadly. But all experiments use Java only (acknowledged in Section 8), two courses, single-file assignments, and primarily GPT-4o-mini. The Limitations section lists these boundaries but the title and abstract are unbounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. For example: the QS rubric improvement could be partly due to the rubric being more detailed (more information provided to the LLM) rather than question-specificity per se. The paper does not consider confounds or alternative interpretations."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures what it claims: agreement between LLM and human grading. The metrics (correlation, ICC, Kappa, Leniency) are appropriate for measuring grading quality. The gap between measurement and framing is small — they claim better LLM-based code evaluation and measure LLM-human grading agreement."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'OpenAI's GPT-4o-mini' and 'Anthropic AI's Claude 3.7 Sonnet' (Section 5.1) without snapshot dates or API version identifiers. These are marketing names that do not pin a specific model version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for CRE, PRE, and EME techniques provided in Appendix A, including exact system messages and output format specifications."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters reported for any of the LLM calls. Section 5.1 mentions only model names. These settings significantly affect LLM output and grading consistency."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "CRE's architecture (LLM grader + compiler agent + recursive mark calculator) is described in Section 4.2 and Figure 9. PRE's per-criterion approach in Section 4.3. EME's ensemble workflow with approach identification and voting in Section 4.4 and Figure 10."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 documents scaling (linear rescaling from QA 4-point to QS 35-point scale) and binning (5 bins at 20th/40th/60th/80th percentiles for Cohen's Kappa). Section 3 describes data selection criteria for both datasets."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Limitations and Future Work' presents multiple specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 lists specific threats: single LLM (GPT-4o), Java-only evaluation, limited to two intermediary courses with single-file assignments, unexplored rubric granularity effects, no open-source vs closed-source comparison."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states what was NOT tested: other programming languages, other LLMs, multi-file assignments, other rubric granularities, open-source LLMs. These are specific exclusions, not generic disclaimers."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Full dataset available on HuggingFace (https://huggingface.co/datasets/BITS-Pilani-GRC/RubricEval) including student code, rubrics, grades, and feedback."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1: OOP data from Fall 2024 exam at BITS Pilani, 350 total submissions, 80 selected across 4 score ranges. Section 3.2: DSA data from GFG website, 25 problems across 9 topics and 3 difficulty levels, 6 submissions per problem."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "OOP submissions: from undergraduate sophomores in a programming exam, 20 randomly selected from each of 4 score ranges. DSA submissions: 6 per problem (3 correct, 1 wrong, 1 TLE, 1 compilation error) from GFG. Human graders described as 'final-year students with extensive experience in programming courses.'"
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented: OOP (350 submissions → 80 selected by score stratification → TA grading with consensus → LLM evaluation). DSA (25 GFG problems → 6 submissions each → rubric design → collaborative human grading → LLM evaluation). Scaling and binning steps in Section 5.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 10: 'This research was carried out in part with support from the New Faculty Seed Grant, Birla Institute of Technology and Science (BITS), Pilani (Grant Ref. N4/24/1004).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors list BITS Pilani as affiliation in the paper header. No evaluated product is affiliated with the authors' institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The BITS Pilani seed grant is an institutional research fund with no financial stake in whether question-specific rubrics outperform question-agnostic ones."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for GPT-4o-mini or Claude 3.7 Sonnet. This is relevant because DSA problems from GFG are widely available online and could be in the models' training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLMs might have encountered these problems, solutions, or rubrics during training. GFG is a popular website whose content is likely in LLM training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "DSA problems are sourced from Geeks for Geeks, a widely-scraped website. The LLMs almost certainly encountered these problems and editorial solutions during training, which could bias their grading. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the research design. Student code is used as evaluation data, and human graders produce ground truth, but neither are experimental subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the experimental design. The study evaluates automated grading approaches against human-produced ground truth."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Student submissions and human graders are data sources, not study subjects."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Submission selection criteria are documented under data_integrity."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in an experimental study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in an experimental study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "PRE is described as 'resource-intensive due to multiple API calls per student solution' (Section 4.3) but no actual costs, token counts, or per-example latencies are reported. Section 7.1 mentions cost trade-offs qualitatively without quantification."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total API spend, token consumption, or computational budget reported despite using commercial LLM APIs (GPT-4o-mini, Claude 3.7 Sonnet) across hundreds of evaluations."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or sensitivity analysis across runs. LLM outputs are non-deterministic but results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation runs per technique is not stated. It is unclear whether results are from a single LLM call per submission or averaged across multiple calls."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. The prompts (Appendix A) were presumably developed iteratively, but no search process or number of iterations is documented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of how the final prompts or technique configurations were selected. The prompts in Appendix A appear to be the result of development/tuning but the selection process is not described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many pairwise comparisons across techniques and metrics without any multiple comparison correction. Seven techniques compared on seven metrics across two datasets."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose CRE, PRE, and EME, implement their own versions of baselines (CodeJudge), and compare without acknowledging author-evaluation bias. No independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "PRE is noted as more expensive than CRE qualitatively, but no performance-vs-cost curves or quantified cost comparisons are provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "Human grading by 'final-year students with extensive programming experience' is used as ground truth without questioning its validity. Inter-rater reliability between the two human graders is not reported. Whether undergrad graders constitute an appropriate gold standard is not discussed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Cross-technique comparisons (CRE vs CodeJudge vs FPM) confound scaffold architecture with rubric type. While EME-QA vs EME-QS controls for scaffold, the overall comparison table mixes scaffold and rubric effects without acknowledging this confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "DSA problems are from Geeks for Geeks, a website whose content predates the training of GPT-4o-mini and Claude 3.7 Sonnet. The LLMs likely encountered these problems and editorial solutions during training. This temporal overlap is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The LLM grader receives the model solution alongside student code. Whether this constitutes feature leakage (the answer is in the input) vs. legitimate reference information is not discussed. The impact of providing vs. withholding the model solution is not tested."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the structural similarity between GFG problems and LLM training data affects results. The OOP dataset (from a specific exam) is less likely contaminated, but no independence analysis is performed for either dataset."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method applied. No canary strings, membership inference, or decontamination analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "Question-specific rubrics substantially outperform question-agnostic rubrics for LLM-based code evaluation, especially on algorithmically diverse problems.",
       "evidence": "Table 3 (DSA): EME-QS achieves Pearson 0.825, ICC3 0.819, Kappa 0.646 vs EME-QA at 0.562, 0.560, 0.156. Table 2 (OOP): comparable performance between EME-QS and EME-QA.",
       "supported": "moderate"
     },
     {
       "claim": "CRE achieves near-human leniency while PRE is significantly harsher.",
       "evidence": "Table 2: CRE (OpenAI) leniency = 0.082 vs PRE leniency = -0.329. PRE's single-criterion prompts force zero-or-full marking, slashing average scores by 11.5/35 marks (Section 6.2.3).",
       "supported": "strong"
     },
     {
       "claim": "LLM-based techniques outperform similarity-based approaches (CodeBERTScore) significantly.",
       "evidence": "Tables 2 and 3: CodeBERTScore achieves Pearson 0.354/0.126, Kappa 0.241/0.010 on OOP/DSA, while all LLM-based techniques achieve substantially higher values.",
       "supported": "strong"
     },
     {
       "claim": "Providing a rubric improves LLM grading performance compared to no-rubric approaches.",
       "evidence": "Tables 2 and 3: CodeJudge (no rubric) achieves Pearson 0.717/0.423 vs EME-QA 0.904/0.562 and EME-QS 0.900/0.825 on OOP/DSA.",
       "supported": "strong"
     },
     {
       "claim": "EME performance improves with larger model parameter size and ensemble size (up to 3-4 models).",
       "evidence": "Section 6.2.4 describes these trends qualitatively but provides no quantitative data or tables for ensemble size or model size experiments.",
       "supported": "weak"
     },
     {
       "claim": "For homogeneous implementation-oriented questions (OOP), question-agnostic rubrics achieve comparable results to question-specific rubrics.",
       "evidence": "Table 2: EME-QA (Pearson 0.904, ICC3 0.904, Kappa 0.512) vs EME-QS (0.900, 0.900, 0.545) on OOP dataset — very similar performance.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims ('significantly enhance', 'outperform') are made by visually comparing metric values in tables. No significance tests are applied to any comparison, despite sample sizes of 80 and 150."
     },
     {
       "flag": "No variance or reproducibility assessment",
       "detail": "LLM outputs are non-deterministic, but all results appear to be from single runs. No standard deviations, confidence intervals, or multi-run analysis. Results could shift substantially on a second run."
     },
     {
       "flag": "Possible training data contamination",
       "detail": "DSA problems are from Geeks for Geeks, a widely-scraped programming website. GPT-4o-mini and Claude 3.7 Sonnet have almost certainly seen these problems and editorial solutions during training, potentially inflating grading accuracy. The OOP dataset (Fall 2024 BITS Pilani exam) is less likely contaminated."
     },
     {
       "flag": "Human ground truth validity unverified",
       "detail": "Ground truth grades produced by 'final-year students with extensive programming experience' — not expert instructors. Inter-rater reliability between the two human graders is not reported. The quality of the ground truth is assumed rather than validated."
     },
     {
       "flag": "No dev/test split for prompt development",
       "detail": "Prompts were presumably iterated and refined, but all evaluation is on the same data used for development. No held-out test set to guard against overfitting the prompts to the evaluation data."
     },
     {
       "flag": "Missing quantitative ensemble analysis",
       "detail": "Section 6.2.4 claims performance varies with model size and ensemble size but provides no quantitative data, tables, or figures to support these claims — only qualitative descriptions."
     }
   ],
   "cited_papers": [
     {
       "title": "Generative AI for Programming Education: Benchmarking ChatGPT, GPT-4, and Human Tutors",
       "authors": ["Tung Phung", "Victor-Alexandru Pădurean", "José Cambronero", "Sumit Gulwani", "Tobias Kohn", "Rupak Majumdar", "Adish Singla", "Gustavo Soares"],
       "year": 2023,
       "arxiv_id": "2306.17156",
       "relevance": "Benchmark comparison of LLMs vs human tutors for programming education, used question-agnostic FPM technique that this paper improves upon."
     },
     {
       "title": "CodEv: An Automated Grading Framework Leveraging Large Language Models for Consistent and Constructive Feedback",
       "authors": ["En-Qi Tseng", "Pei-Cing Huang", "Chan Hsu", "Peng-Yi Wu", "Chan-Tung Ku", "Yihuang Kang"],
       "year": 2024,
       "doi": "10.1109/bigdata62323.2024.10825949",
       "relevance": "LLM ensemble-based code evaluation framework using question-agnostic rubrics, directly compared against in this work."
     },
     {
       "title": "CodeJudge: Evaluating Code Generation with Large Language Models",
       "authors": ["Weixi Tong", "Tianyi Zhang"],
       "year": 2024,
       "arxiv_id": "2410.02184",
       "relevance": "LLM-based code evaluation using slow-thinking decomposition, used as a no-rubric baseline."
     },
     {
       "title": "ICE-Score: Instructing Large Language Models to Evaluate Code",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2304.14317",
       "relevance": "Early work on using LLMs for code evaluation without test cases, showing limited correlation with human judgment."
     },
     {
       "title": "CodeBERTScore: Evaluating Code Generation with Pretrained Models of Code",
       "authors": ["Shuyan Zhou", "Uri Alon", "Sumit Agarwal", "Graham Neubig"],
       "year": 2023,
       "arxiv_id": "2302.05527",
       "relevance": "Semantic code evaluation metric using BERT embeddings, used as a non-LLM baseline showing limitations of similarity-based approaches."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, foundational model family used in the evaluation (GPT-4o-mini)."
     },
     {
       "title": "Automating Autograding: Large Language Models as Test Suite Generators for Introductory Programming",
       "authors": ["Umar Alkafaween", "Ibrahim Albluwi", "Paul Denny"],
       "year": 2024,
       "doi": "10.1111/jcal.13100",
       "relevance": "LLM-based approach to automated grading through test suite generation for programming courses."
     },
     {
       "title": "Evaluating Language Models for Generating and Judging Programming Feedback",
       "authors": ["Charles Koutcheme", "Nicola Dainese", "Sami Sarsa", "Arto Hellas", "Juho Leinonen", "Syed Ashraf", "Paul Denny"],
       "year": 2025,
       "doi": "10.1145/3641554.3701791",
       "relevance": "Evaluation of LLMs for generating and judging programming feedback quality, directly relevant to LLM-based code assessment."
     },
     {
       "title": "Grade Like a Human: Rethinking Automated Assessment with Large Language Models",
       "authors": ["Wenjing Xie", "Juxin Niu", "Chun Jason Xue", "Nan Guan"],
       "year": 2024,
       "arxiv_id": "2405.19694",
       "relevance": "Multi-agent LLM grading system that creates context-aware rubrics for short-answer questions, closest prior work on rubric-based LLM assessment."
     },
     {
       "title": "Large Language Models (GPT) for automating feedback on programming assignments",
       "authors": ["Maciej Pankiewicz", "Ryan S. Baker"],
       "year": 2023,
       "arxiv_id": "2307.00150",
       "relevance": "Early exploration of GPT models for automated programming feedback generation."
     }
   ]
 }