ai-research-survey

scan.json (31888B)

---

 {
   "paper": {
     "title": "On the Effectiveness of LLM-as-a-judge for Code Generation and Summarization",
     "authors": [
       "Giuseppe Crupi",
       "Rosalia Tufano",
       "Alejandro Velasco",
       "Antonio Mastropaolo",
       "Denys Poshyvanyk",
       "Gabriele Bavota"
     ],
     "year": 2025,
     "venue": "IEEE Transactions on Software Engineering",
     "arxiv_id": "2507.16587",
     "doi": "10.1109/TSE.2025.3586082"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper references a replication package at [1] with a GitHub URL (https://github.com/crupig/LLMs-as-a-judge-for-SE-tse_RP) mentioned in the references section and cited throughout the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we built (and make publicly available [1]) our own dataset that features human judgments of 1,163 summaries' (Section 2.2.2). CoderEval is also a public benchmark. All data is in the replication package."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification (requirements.txt, Dockerfile, library versions) is mentioned. The paper mentions using Hugging Face inference endpoints and ChatGPT APIs but provides no reproducible environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. The replication package is referenced but no README with commands or a 'Reproducing Results' section is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (Cohen's Kappa, Krippendorff's alpha, accuracy, bias coefficients) are reported as point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Mann-Whitney U tests with Benjamini-Hochberg correction are used for the self-bias analysis (Section 2.4, Tables 3 and 6). Adjusted p-values are reported at multiple significance levels."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cliff's delta effect sizes are reported alongside statistical tests (Tables 3 and 6), with interpretation thresholds defined: negligible, small, medium, large (Section 2.4)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for sample sizes. The 184 Java and 190 Python code generation problems, and 198 functions for summarization, are used without justifying adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation reported across experimental runs. LLM outputs are stochastic, but all results appear from single runs with no spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Eight LLMs of varying sizes are compared against each other and against ground truth (test execution for code generation, human judgments for summarization). Four different prompting strategies are compared (Table 2, Table 5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "For a paper published in July 2025, the models are dated. GPT-4-turbo was the strongest model tested, but GPT-4o (May 2024), Claude 3/3.5, Llama 3, and other 2024-2025 models are absent. No justification is given for the model selection timeframe."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates along multiple dimensions: four prompt strategies (Table 2, Table 5), self-contained vs. non-self-contained functions (Section 3.1), and the mutant injection + semantic equivalence study (Section 3.1.2, Fig. 2)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: Cohen's Kappa, confusion matrices (TP/TN/FP/FN rates, accuracy), bias coefficient for code generation; Krippendorff's alpha and scatterplots for summarization."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Nine human judges independently evaluated 1,163 code summaries across three quality criteria (content adequacy, conciseness, fluency & understandability) on a 1-5 scale, with each summary assessed by three judges (Section 2.2.2)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Prompts were developed using 'toy examples' separate from the evaluation data. The main evaluation was conducted on CoderEval (code generation) and the purpose-built summarization dataset, which were not used for prompt tuning (Section 2.3.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by language (Java/Python), by model, by model family, by prompt strategy, and by function type (self-contained vs. dependent). Confusion matrices per model (Fig. 1) and per-criterion scores for summarization (Table 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1.2 provides detailed manual analysis of false positives (37% uncaught wrong behavior, 32% coding context, 27% ambiguous requirements) and false negatives (33% hallucination, 19% code misunderstanding). Table 1 documents judging failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The main findings are largely negative: smaller LLMs completely fail at judging, GPT-4-turbo misjudges 50% of wrong Java implementations. The paper explicitly reports what doesn't work. DeepSeek Coder family was excluded from summarization entirely due to inability to perform the task."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: GPT-4-turbo as best judge (Tables 2, 5), smaller LLMs struggling (Tables 1, 2, 5), frequent misjudgments of code correctness (Fig. 1, 50% FP rate), moderate agreement on summarization (Table 5, α=0.58-0.63 for content adequacy)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper primarily makes comparative/descriptive claims about judging effectiveness rather than strong causal claims. Where causal reasoning is used (e.g., coding context as a factor in misjudgments), it is tested empirically by analyzing self-contained functions separately (Section 3.1)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 explicitly states: 'the generalizability of our findings is capped by (i) the two code-related tasks subject of the study and (ii) the focus on the Java and Python programming languages. Differentiated replications can help to corroborate/contradict our findings.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses construct validity (tests as proxy for correctness), prompt impact, and manual analysis subjectivity. Section 3.1 investigates coding context as an alternative explanation for misjudgments. Section 3.1.2 systematically analyzes reasons behind false judgments."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 explicitly acknowledges: 'Using tests as a proxy for code correctness is a limitation of our study.' They also document cases where the proxy fails (unreliable test suites, Section 2.2.1) and excluded problematic cases."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are named as 'GPT-3.5-turbo' and 'GPT-4-turbo' without snapshot dates or API versions. DeepSeek Coder and CodeLlama sizes are specified (1.3B, 6.7B, 33B; 7B, 13B, 34B) but exact checkpoint versions are not given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided for the zero-shot code generation prompt (Section 2.3.1), the automated CoT prompts (Section 2.3.1), and the zero-shot summarization prompt (Section 2.3.2). Additional prompts are in the replication package [1]."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for any model. These settings significantly affect LLM output and are essential for reproducibility."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. LLMs are prompted directly via API calls or Hugging Face inference endpoints."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Detailed quality assurance pipeline for CoderEval documented in Section 2.2.1: 230 Java → 210 (excluded 20 failing targets) → 201 (excluded 9 empty-body passes) → 184 (excluded 17 dummy-function passes). Similar filtering for Python: 230 → 191 → 190. Summarization dataset construction also documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 'Threats to Validity' is a dedicated section covering construct, internal, and external validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses specific threats: test suites as imperfect proxies for correctness (with mitigation), subjectivity in manual analysis (mitigated by multiple evaluators), prompt sensitivity (tested with four variants), and language/task scope limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 explicitly states what the results do NOT show: 'the generalizability of our findings is capped by (i) the two code-related tasks subject of the study and (ii) the focus on the Java and Python programming languages.' Future work specifies additional tasks and fine-tuning as unexplored directions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package [1] is referenced as containing all data: prompts, judgments, generated code, human evaluations. The summarization dataset is explicitly stated as 'publicly available [1]' (Section 2.2.2)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail: CoderEval benchmark selection and quality filtering (Section 2.2.1), summarization dataset construction from top-100 longest functions per language (Section 2.2.2), LLM output extraction using lizard code analyzer (Section 2.3.1)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The 9 human judges are characterized by qualifications (Master's/PhD, years of experience) but how they were recruited is not described. No information on recruitment channels or whether they were lab members, students, or external participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: CoderEval quality filtering with counts at each stage (Section 2.2.1), code generation and extraction (Section 2.3.1), judgment collection with manual verification (Section 2.3.1), and total counts (80,556 code gen judgments, 22,304 summarization judgments)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper. A 6-author paper from two universities likely received institutional funding but this is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: SEART @ Software Institute, Università della Svizzera italiana (Crupi, Tufano, Bavota) and W&M (Velasco, Mastropaolo, Poshyvanyk). None of the authors are affiliated with the companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding disclosure does not establish that no funding exists."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the eight LLMs. This is relevant because CoderEval benchmark problems could have appeared in the training data of GPT-4-turbo, GPT-3.5-turbo, and other models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CoderEval problems or their solutions appeared in any model's training data. CoderEval was published at ICSE'24 and its code/solutions are publicly available."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Benchmark contamination is not addressed. CoderEval problems are from open-source repositories and were publicly available before GPT-4-turbo's training cutoff, creating contamination risk for both the generation and judging tasks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the human evaluation study with 9 judges."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the human evaluation study."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2.2 reports: 'All nine judges have a Master's degree in Informatics or Computer Science, four of them have a Ph.D. in Software Engineering. On average, they have 5.8 years of experience (min=1, max=17) in Java programming and 6.9 in Python programming (min=4, max=10).'"
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "The judges are described as having 'code summarization background' but no formal inclusion/exclusion criteria are stated. No screening process is described."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a rating/evaluation study, not an experimental study with treatment and control conditions. The human judges are evaluators, not experimental participants assigned to conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an evaluation/rating study rather than a randomized experiment. While blinding judges to summary source (human vs. LLM) would have been methodologically valuable, the schema marks this as NA for non-experimental studies."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition or dropout information is reported. The paper states 9 judges and 3,489 total judgments expected but does not confirm whether all judgments were completed or if any judges dropped out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported despite running 80,556 code generation judgments and 22,304 summarization judgments across eight LLMs, including commercial API calls to GPT-3.5-turbo and GPT-4-turbo."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper used Hugging Face inference endpoints for open models and ChatGPT APIs for GPT models, but neither API costs nor GPU hours are reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or runs. LLM outputs are stochastic, but results appear to be from single runs without seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It appears each judgment was obtained from a single LLM call, but this is never explicitly confirmed."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Four prompts are compared, but the number of prompt variants tested during the trial-and-error development phase is not disclosed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 explicitly justifies the selection criterion: 'Since for both tasks there was one judge LLM which was the clear winner independently from the used prompt (i.e., GPT-4-turbo), we selected as best-performing prompt the one ensuring the best performance on it.' Results for all four prompts are shown (Tables 2, 5)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": true,
         "justification": "Benjamini-Hochberg correction is applied to adjust p-values for multiple comparisons in the self-bias analysis (Section 2.4, Tables 3, 6)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss author-evaluation bias. The prompts, extraction scripts, and experimental design were developed by the authors, but no acknowledgment of potential bias in their own evaluation methodology is made."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of performance as a function of compute budget. Larger models (GPT-4-turbo with estimated >1.5T parameters) are compared with smaller ones (1.3B) without discussing compute cost implications, despite the paper noting that 'the larger the language model, the higher its inference (judgment) cost.'"
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2.1 extensively analyzes CoderEval's construct validity: verifying target functions pass tests, checking for trivial passing implementations (empty functions, dummy returns), and excluding 76 of 460 problems with unreliable test suites. Section 4 discusses tests as proxy for correctness."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. LLMs are prompted directly without agentic scaffolding or tool use."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. CoderEval functions are from open-source projects that could have been in training data for GPT-4-turbo and other models. The temporal relationship between benchmark creation and model training is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the judging prompts (description + signature + candidate) provide information that could trigger memorized solutions from training data."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CoderEval problems or their solutions overlap with LLM training data. The benchmark draws from open-source repositories that are likely in LLM training corpora."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, n-gram overlap analysis, or decontamination procedures are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4-turbo is the best judge among the eight tested LLMs for both code generation and code summarization tasks.",
       "evidence": "Table 2 shows GPT-4 achieves the highest Cohen's Kappa for code generation (0.21 Java, 0.10 Python with automated CoT). Table 5 shows GPT-4 achieves highest Krippendorff's alpha for code summarization (0.58 Java, 0.63 Python for content adequacy). Section 3.",
       "supported": "strong"
     },
     {
       "claim": "Smaller LLMs (1-13B parameters) cannot effectively act as judges for code generation or summarization.",
       "evidence": "Table 2: DeepSeek Coder 1.3B, 6.7B and CodeLlama 7B achieve negative or near-zero Kappa scores. Table 5: CodeLlama 7B, 13B, 34B all achieve negative Krippendorff's alpha for most criteria. DeepSeek Coder family excluded entirely from summarization study due to inability to perform the task.",
       "supported": "strong"
     },
     {
       "claim": "Even GPT-4-turbo frequently misjudges code correctness, especially failing to identify incorrect implementations — 50% of wrong Java implementations are misjudged as correct.",
       "evidence": "Fig. 1 confusion matrices show GPT-4 correctly classifies 72% of correct Java implementations but misjudges 50% of wrong ones as correct (50% false positive rate for wrong code).",
       "supported": "strong"
     },
     {
       "claim": "GPT-4-turbo achieves moderate agreement with humans for judging code summary quality, particularly content adequacy.",
       "evidence": "Table 5: Krippendorff's alpha for content adequacy is 0.58 (Java) and 0.63 (Python), which falls in the moderate agreement range. Conciseness alpha: 0.40 Java, 0.36 Python.",
       "supported": "strong"
     },
     {
       "claim": "LLMs are not significantly self-biased in code generation judging, with the exception of GPT-4 showing negligible self-bias.",
       "evidence": "Table 3: Mann-Whitney tests show no significant difference for most LLMs when judging own vs. others' code. GPT-4 shows significant but negligible effect size (p<0.01, Cliff's d=Negligible).",
       "supported": "strong"
     },
     {
       "claim": "LLMs systematically underestimate the correctness of human-written code compared to LLM-generated code.",
       "evidence": "Table 3: Human-written code has average bias coefficient of -0.37 (underestimated), while all LLM-generated code has positive coefficients (overestimated). All statistical tests show significant difference with large effect size for human vs. own code.",
       "supported": "strong"
     },
     {
       "claim": "Artificial hallucination is the most common cause of false negative judgments (33%), with LLMs commenting on non-existent bugs or missing requirements that were actually implemented.",
       "evidence": "Section 3.1.2: Manual analysis of false negatives found 33% were due to artificial hallucination, 19% due to misunderstanding of code statements.",
       "supported": "moderate"
     },
     {
       "claim": "Prompt choice has some impact on judging performance but does not substantially change overall findings or the identification of the best model.",
       "evidence": "Tables 2 and 5 show results across four prompting strategies. Kappa and Krippendorff's alpha values are relatively stable across prompts for each model, and GPT-4 remains the best performer across all prompts.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "GPT-4-turbo is the best LLM judge for both code generation and summarization among eight tested models, but still frequently misjudges code correctness (50% of wrong Java implementations judged as correct). Smaller LLMs (1-33B parameters) are largely unable to perform judging tasks. For code summarization, GPT-4-turbo achieves moderate agreement with human judgments (Krippendorff's alpha 0.58-0.63 for content adequacy). LLMs systematically underestimate human-written code quality while overestimating LLM-generated code quality, with large effect sizes.",
   "red_flags": [
     {
       "flag": "Outdated model selection",
       "detail": "Published July 2025 but tests only models available in 2023-early 2024 (GPT-4-turbo, CodeLlama, DeepSeek Coder v1). Major 2024-2025 models (GPT-4o, Claude 3/3.5, Llama 3, DeepSeek V2/V3, Gemini) are completely absent, significantly limiting the relevance of findings."
     },
     {
       "flag": "No hyperparameters reported",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any of the eight LLMs. These settings substantially affect output quality and are essential for reproducibility of the 100,000+ judgments collected."
     },
     {
       "flag": "Single-run stochastic results",
       "detail": "LLM outputs are stochastic, but no multiple runs, seed sensitivity, or variance across runs is reported. A different random seed could yield different judgments, especially for borderline cases."
     },
     {
       "flag": "No contamination analysis",
       "detail": "CoderEval functions are from open-source projects likely in LLM training data. Models may have memorized solutions, affecting both code generation and judging accuracy. This confound is not discussed."
     },
     {
       "flag": "Small convenience sample of human judges",
       "detail": "Only 9 human judges with unspecified recruitment method. Likely convenience sample from the authors' institutions. No IRB approval mentioned despite involving human participants."
     }
   ],
   "cited_papers": [
     {
       "title": "CoderEval: A Benchmark of Pragmatic Code Generation with Generative Pre-trained Models",
       "authors": ["H. Yu", "B. Shen", "D. Ran", "J. Zhang", "Q. Zhang", "Y. Ma", "G. Liang", "Y. Li", "Q. Wang", "T. Xie"],
       "year": 2024,
       "relevance": "Primary benchmark used for evaluating LLM code generation and judging capabilities; ICSE'24 paper with 460 code generation problems in Java and Python."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced the HumanEval benchmark and pass@k metric, foundational for LLM code generation evaluation."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["B. Rozière"],
       "year": 2024,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM family (7B-34B) evaluated as both code generators and judges in this study."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "authors": ["D. Guo", "Q. Zhu", "D. Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Code-focused LLM family (1.3B-33B) evaluated as judges, found unable to perform summarization judging tasks."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["L. Zheng", "W.-L. Chiang", "Y. Sheng"],
       "year": 2023,
       "arxiv_id": "2306.05685",
       "relevance": "Foundational work on LLM-as-a-judge paradigm identifying positional bias, verbosity bias, self-enhancement bias, and limited reasoning ability."
     },
     {
       "title": "ICE-score: Instructing Large Language Models to Evaluate Code",
       "authors": ["T. Y. Zhuo"],
       "year": 2024,
       "relevance": "Prior work using GPT-3.5-turbo as a judge for code implementations on HumanEval-X; reports weak-to-moderate correlations with test outcomes."
     },
     {
       "title": "CodeJudge: Evaluating Code Generation with Large Language Models",
       "authors": ["W. Tong", "T. Zhang"],
       "year": 2024,
       "relevance": "GPT-3.5-based code correctness judge using slow-thinking prompt strategy; reports 54-57% accuracy on complex benchmarks, aligned with this paper's findings."
     },
     {
       "title": "CodeUltraFeedback: An LLM-as-a-Judge Dataset for Aligning Large Language Models to Coding Preferences",
       "authors": ["M. Weyssow", "A. Kamanda", "H. Sahraoui"],
       "year": 2024,
       "relevance": "Proposes using LLM-as-a-judge for evaluating non-functional code requirements; reports LLMs agree that GPT-based models produce superior solutions."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["T. Y. Zhuo", "M. C. Vu", "J. Chim"],
       "year": 2024,
       "relevance": "Comprehensive code generation benchmark used in CodeJudge evaluation; features complex coding problems beyond HumanEval difficulty."
     },
     {
       "title": "Reassessing Automatic Evaluation Metrics for Code Summarization Tasks",
       "authors": ["D. Roy", "S. Fakhoury", "V. Arnaoudova"],
       "year": 2021,
       "relevance": "Large-scale dataset of human code summary evaluations (6,253 evaluations by 226 developers) showing low inter-rater agreement; motivates the need for better evaluation methods."
     },
     {
       "title": "On the Limitations of Fine-tuned Judge Models for LLM Evaluation",
       "authors": ["H. Huang", "Y. Qu", "H. Zhou"],
       "year": 2024,
       "arxiv_id": "2403.02839",
       "relevance": "Studies fine-tuning LLMs specifically for judging tasks, showing limitations in generalizability, fairness, and scalability of specialized judge models."
     },
     {
       "title": "Evaluating Language Models for Generating and Judging Programming Feedback",
       "authors": ["C. Koutcheme", "N. Dainese", "A. Hellas"],
       "year": 2024,
       "arxiv_id": "2407.04873",
       "relevance": "Evaluates LLMs for judging programming assignments at beginner level; identifies hallucination as a key failure mode in code judging."
     }
   ]
 }