ai-research-survey

scan.json (28257B)

---

 {
   "paper": {
     "title": "Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering",
     "authors": ["Ruiqi Wang", "Jiyu Guo", "Cuiyun Gao", "Guodong Fan", "Chun Yong Chong", "Xin Xia"],
     "year": 2025,
     "venue": "Proc. ACM Softw. Eng. (ISSTA)",
     "arxiv_id": "2502.06193",
     "doi": "10.1145/3728963"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our source code and data is publicly available at [46]' (Section Data Availability) and provides a GitHub URL: https://github.com/BackOnTruck/llm-judge-empirical."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at [46] is described as containing both source code and data. The paper also uses publicly available datasets (CodeTransOcean, CodeXGLUE, ComplexCodeEval) and describes curating a meta-evaluation dataset of 450 scored responses that is included in the package."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'an Ubuntu 20.04 server with two Intel Xeon Platinum 8276L CPUs, four NVIDIA A100-40GB GPUs, and 256 GB RAM' and that vLLM was used, but there is no requirements.txt, Dockerfile, or detailed dependency version listing in the paper. Hardware is described but software environment is not specified with enough detail to recreate it."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the methodology in detail (datasets, model selection, scoring procedure) but does not provide step-by-step reproduction instructions such as commands to run or a README with a reproduction guide within the paper itself. The repository may contain these, but the paper does not describe them."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports correlation coefficients (Spearman's rho, Pearson's R, Kendall's tau) as point estimates throughout Tables 3 and 6. No confidence intervals or error bars are provided for any of these metrics."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states 'We also check if the p-value of each correlation coefficient in RQ1 is smaller than 0.05 to ensure a 95% confidence interval.' Table 3 marks coefficients with p > 0.05 in red."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports correlation coefficients themselves as effect size measures (e.g., R = 81.32, rho = 73.67) and provides absolute differences between methods (e.g., 'Δ R = 2.21, 6.03, 3.04 at maximum' for inference strategies). The correlations with baselines provide clear context for the magnitude of improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample of 50 instructions per task (150 total, 450 responses) is not justified with a power analysis or explicit rationale for why 50 is sufficient. No discussion of whether the sample size provides adequate statistical power for the correlation analyses."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report variance or standard deviation across experimental runs. G-Eval uses 20 inference passes and averages them, BatchEval uses 5 rounds, but no variance across these runs is reported. Results appear to be single-run numbers for most methods."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes five conventional metrics as baselines (BLEU, ROUGE-L, METEOR, ChrF++, CrystalBLEU) and compares them against all LLM-as-a-judge methods in Table 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The conventional metrics represent the standard evaluation practice in SE. The LLM-as-a-judge methods include recent approaches (G-Eval 2023, BatchEval 2024, Prometheus 2 2024) and use GPT-4o (2024-08-06 version). The baselines are appropriate for the comparison being made."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study design effectively ablates multiple dimensions: method category (embedding, probability, output-based), LLM size (small <50B vs large >100B in Table 5), inference strategy (vanilla vs G-Eval vs BatchEval), and SFT vs non-SFT. RQ1-RQ3 systematically vary these components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three correlation metrics are used for individual scoring: Spearman's rho, Pearson's R, and Kendall's tau (Table 3). For pairwise comparison, Accuracy and Agreement are both reported (Table 6)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human evaluation is central to the study. Two human evaluators scored all 450 responses on two aspects plus an overall score. Inter-annotator agreement is reported (footnote 10): Spearman's rho of 83.07, 75.42, 74.20 across tasks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses test sets from existing benchmarks: 'When training, validation, and test sets are available, we only adopt the test set for our evaluation' (Section 4.1.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task (code translation, code generation, code summarization) in Tables 3 and 6, by method category (Table 4), and by LLM size (Table 5). This provides granular analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 presents a detailed case study of a failing case (code summarization) where GPT-4o assigns a perfect score while the human score is 2.75, with analysis of why the LLM failed (verbosity bias). The paper also discusses failure modes across tasks."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports substantial negative results: LLM-as-a-judge methods 'completely defeated by conventional metrics' in code summarization (Section 5.1), pairwise comparison 'fail to deliver satisfactory and consistent comparison performance' (Finding 5), inference strategies provide only 'marginal improvement' (Section 5.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims output-based methods reach Pearson correlation of 81.32 and 68.51, outperforming ChrF++ at 34.23 and 64.92. These numbers match Table 3. The hedged conclusion 'can potentially replace human evaluations in certain SE tasks' is supported by the mixed task-dependent results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper primarily makes correlational claims ('methods X correlate with human scores at Y'). When it makes explanatory claims (e.g., poor summarization performance due to verbosity bias), these are supported by the case study analysis in Section 6.1. The ablation-like design (varying method type, LLM size, inference strategy) supports the component-level claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Can LLMs Replace Human Evaluators?' and the abstract claim about replacing 'human evaluations in certain SE tasks' are broad. The study tests only 3 SE tasks with 50 instructions each, using specific datasets and a limited set of programming languages (Java, Python, C, C++). The paper does not adequately bound its findings to these specific tasks and settings; the title and framing suggest broader applicability."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations for observed results: poor conventional metrics in translation attributed to reference-response divergence (Section 5.1), poor LLM performance in summarization attributed to verbosity bias (Section 6.1), SFT model underperformance attributed to limited parameters and NLP-SE misalignment (Section 5.1)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "For GPT-4o, a version date is given: '2024-08-06 version' (footnote 8). However, for most other models, only family names and sizes are given (e.g., 'CodeLlama-Instruct 7/13/34B', 'DeepSeek-Coder-V2-Lite 16B', 'DeepSeek-V2.5'). These are marketing names without specific checkpoint identifiers or snapshot dates. Crucially, 'DeepSeek-V2.5' and 'Codestral-v0.1' lack precise version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'The full prompts are available in our repository [46]' and 'the remaining aspects and criteria can be found in our repository' (Section 4.1.3). Only one example evaluation rubric is shown in the paper itself (readability for code summarization). The actual prompts used for LLM-as-a-judge methods are not provided in the paper text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature settings are reported for key methods: Vanilla uses greedy decoding (temperature 0), G-Eval uses temperature 1.0 with 20 inference passes, BatchEval uses temperature 0.2 with batch size 10 and 5 rounds (Section 4.2.4). Token generation limits are specified (3072 tokens for response generation, 1024 for judgments)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The methods are direct prompt-based evaluations (single or multi-pass inference), not agent-based workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 describes filtering by programming language (Java, Python, C, C++), token length limits (1536/1536/1024), reselection of instructions for code summarization (reference summaries with at least 15 tokens), and manual examination. Section 4.1.2 describes dependency augmentation with GPT-4o for ComplexCodeEval. The preprocessing pipeline is well-documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. The Discussion (Section 6) covers implications and case studies but does not systematically discuss limitations. The Conclusion (Section 7) is brief and does not contain substantive limitation discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. There is no analysis of how the small sample size (50 per task), the choice of only two human evaluators, the restriction to specific programming languages, or the use of only GPT-4o for output-based inference strategies might affect the conclusions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound the generalizability to the specific 3 tasks, 4 programming languages, or 50 instructions per task. The broad title and conclusions suggest generality beyond what was tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at [46] (https://github.com/BackOnTruck/llm-judge-empirical) contains the source code and data, including the 450 scored responses and human evaluations."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 provides detailed description of how instructions were sampled from three datasets, how responses were generated using 12 LLMs with vLLM, and how human evaluation was conducted with two evaluators using 5-point rubrics on two aspects per task."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Two human evaluators with expertise in the chosen programming languages are involved' (Section 4.1.3) but provides no details about who these evaluators are, how they were recruited, their background, or their relationship to the authors. This lack of detail makes it impossible to assess potential evaluator bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented from instruction collection (Section 4.1.1) through response generation (Section 4.1.2) to manual evaluation (Section 4.1.3) and meta-evaluation (Section 4.3). Filtering steps are described with specific criteria (language, token length, minimum summary length)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed anywhere in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Harbin Institute of Technology Shenzhen, Monash University Malaysia, and Zhejiang University. None of these affiliations appear to create a conflict with the products evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since funding is not disclosed at all, it is impossible to determine whether the funder is independent of the outcome. The absence of any funding disclosure is a concern."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates multiple LLMs (GPT-4o, DeepSeek-V2.5, etc.) as judges and uses LLMs to generate responses. Training cutoff dates are not stated for any of these models. The model release dates are listed in Table 1 but training data cutoffs are not."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation benchmarks (CodeTransOcean, CodeXGLUE, ComplexCodeEval) are all publicly available and could be in the training data of the LLMs used. No discussion of potential train/test overlap is provided."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CodeXGLUE was published in 2021, HumanEval in 2021, and CodeSearchNet in 2019 — all well before the training cutoffs of models like GPT-4o and DeepSeek-V2.5. The paper does not discuss benchmark contamination risk despite using publicly available benchmarks with models that likely trained on them."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The study involves two human evaluators scoring 450 responses. No pre-registration is mentioned."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The study involves human evaluators as participants in the research process. No IRB or ethics board approval is mentioned."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The only characterization of the two human evaluators is that they have 'expertise in the chosen programming languages' (Section 4.1.3). No other demographics (experience level, years of experience, affiliation) are provided."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for selecting the human evaluators are stated. The paper does not explain why these specific evaluators were chosen or what qualifications they have beyond programming language expertise."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study with treatment/control groups assigned to human participants. The human evaluators both scored all responses, so randomization of participant assignment is not applicable."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe whether human evaluators were blinded to which LLM generated each response. Since responses from different LLMs may have identifiable patterns, blinding is relevant. The paper states evaluators received 'the corresponding instruction and the reference answer along with the response' but does not mention whether the generating model identity was hidden."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "With only two evaluators scoring all responses, attrition reporting is not applicable in the traditional sense. Both evaluators completed all evaluations."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses inference cost qualitatively (G-Eval 'greatly increasing inference cost', BatchEval 'more expensive inference due to multi-round evaluation') but does not report actual costs, API spending, tokens consumed, or wall-clock time for any method."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The hardware is listed (4x NVIDIA A100-40GB, Ubuntu 20.04 server) but no total compute budget, GPU hours, API spend, or total inference time is reported."
       }
     }
   },
   "claims": [
     {
       "claim": "Output-based LLM-as-a-judge methods reach Pearson correlation of 81.32 and 68.51 with human scores in code translation and generation respectively, outperforming ChrF++ at 34.23 and 64.92.",
       "evidence": "Table 3 shows BatchEval achieves R=81.32 for code translation and DeepSeek-V2.5 Vanilla achieves R=68.51 for code generation, compared to ChrF++ at R=34.23 and R=64.92 respectively.",
       "supported": "strong"
     },
     {
       "claim": "LLM-as-a-judge methods demonstrate drastically different performance across SE tasks, lacking generalizability.",
       "evidence": "Table 3 shows the same methods that achieve high correlation in translation (R>80) fail in summarization (R<30 for all LLM methods). Finding 1 (Section 5.1) summarizes this pattern.",
       "supported": "strong"
     },
     {
       "claim": "Inference strategies (G-Eval, BatchEval) provide only marginal improvement over vanilla output-based methods with large LLMs for individual scoring.",
       "evidence": "Section 5.1 reports 'ΔR = 2.21, 6.03, 3.04 at maximum' for inference strategies applied to GPT-4o. Table 3 confirms small gaps between GPT-4o vanilla and G-Eval/BatchEval.",
       "supported": "strong"
     },
     {
       "claim": "LLM-as-a-judge methods fail to deliver satisfactory and consistent pairwise comparison performance.",
       "evidence": "Table 6 shows the best accuracy is 65.33 (code translation), with most methods near or below random baseline (33.33). The best-performing methods show Agreement below 25, indicating inconsistency when response order is swapped.",
       "supported": "strong"
     },
     {
       "claim": "Output-based methods with large LLMs (>100B parameters) align well with each other despite differences in models and inference strategies.",
       "evidence": "Table 5 shows Large-Large correlations of rho_min=83.04, rho_max=90.64 for translation and rho_min=68.63, rho_max=88.25 for generation, contrasting with Small-Small correlations below 30.",
       "supported": "strong"
     },
     {
       "claim": "Current LLM-as-a-judge methods can potentially replace human evaluations in certain SE tasks.",
       "evidence": "The evidence supports this only for code translation (R=81.32, near inter-annotator agreement of R=85.86) and partially for code generation (R=68.51 vs inter-annotator R=79.70). For code summarization, all methods fail. The claim is hedged ('potentially', 'certain') but still overstates given only 50 samples per task.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The study evaluates nine LLM-as-a-judge methods against five conventional metrics on three SE tasks (code translation, generation, summarization) using 450 manually scored LLM responses. Output-based methods with large LLMs achieve near-human alignment for code translation (Pearson R=81.32) but fail completely for code summarization (all R<30), revealing strong task dependence. Pairwise comparison performs worse than individual scoring, with high inconsistency when response order is swapped. Inference strategies like G-Eval and BatchEval provide only marginal improvement over vanilla prompting with large LLMs.",
   "red_flags": [
     {
       "flag": "Small sample size without justification",
       "detail": "Only 50 instructions per task (150 total, 450 responses) are used. No power analysis or justification for this sample size is provided. This is small for drawing conclusions about whether LLMs can replace human evaluators."
     },
     {
       "flag": "Only two human evaluators",
       "detail": "The ground truth is based on only two human evaluators whose demographics, qualifications, and recruitment are not described. The inter-annotator agreement varies substantially across tasks (Pearson R from 73.74 to 85.86), and the mean of two evaluators is used as the gold standard."
     },
     {
       "flag": "No limitations or threats-to-validity section",
       "detail": "The paper lacks a dedicated limitations section despite making broad claims about replacing human evaluators. Important limitations (small sample, limited languages, only two annotators, potential contamination) are not discussed."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "Public benchmarks (CodeXGLUE 2021, CodeSearchNet 2019) are used with models trained after their publication. No discussion of whether the judge LLMs or response-generating LLMs may have seen these benchmarks during training."
     },
     {
       "flag": "Broad title and claims relative to evidence scope",
       "detail": "The title asks 'Can LLMs Replace Human Evaluators?' broadly, but tests only 3 tasks in 4 programming languages with 50 instructions each. The scope of evidence is narrow relative to the generality of the claims."
     }
   ],
   "cited_papers": [
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "Foundational work on LLM-as-a-judge methodology that this paper extends to SE tasks."
     },
     {
       "title": "G-Eval: NLG Evaluation using Gpt-4 with Better Human Alignment",
       "authors": ["Yang Liu", "Dan Iter", "Yichong Xu"],
       "year": 2023,
       "relevance": "Key LLM-as-a-judge method evaluated in this study; uses Chain-of-Thought for evaluation."
     },
     {
       "title": "Prometheus 2: An Open Source Language Model Specialized in Evaluating Other Language Models",
       "authors": ["Seungone Kim", "Juyoung Suk", "Shayne Longpre"],
       "year": 2024,
       "arxiv_id": "2405.01535",
       "relevance": "Open-source fine-tuned judge model evaluated in this study for SE task evaluation."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval and Codex; foundational benchmark for code generation evaluation."
     },
     {
       "title": "CodeBERTScore: Evaluating Code Generation with Pretrained Models of Code",
       "authors": ["Shuyan Zhou", "Uri Alon", "Sumit Agarwal", "Graham Neubig"],
       "year": 2023,
       "relevance": "Code-specific evaluation metric adapted from BERTScore; relevant to automated code evaluation methods."
     },
     {
       "title": "DeepSeek-Coder-V2: Breaking the Barrier of Closed-Source Models in Code Intelligence",
       "authors": ["DeepSeek-AI"],
       "year": 2024,
       "arxiv_id": "2406.11931",
       "relevance": "One of the LLMs used both for response generation and as a judge in this study."
     },
     {
       "title": "NoFunEval: Funny How Code LMs Falter on Requirements Beyond Functional Correctness",
       "authors": ["Manav Singhal", "Tushar Aggarwal", "Abhijeet Awasthi"],
       "year": 2024,
       "arxiv_id": "2401.15963",
       "relevance": "Meta-evaluation benchmark for non-functional code quality aspects, related to evaluating code beyond pass/fail."
     },
     {
       "title": "CodeUltraFeedback: An LLM-as-a-Judge Dataset for Aligning Large Language Models to Coding Preferences",
       "authors": ["Martin Weyssow", "Aton Kamanda", "Houari A. Sahraoui"],
       "year": 2024,
       "arxiv_id": "2403.09032",
       "relevance": "Dataset for evaluating LLM alignment with human coding preferences, directly related to LLM-as-a-judge for code."
     },
     {
       "title": "Large Language Models for Software Engineering: A Systematic Literature Review",
       "authors": ["Xinyi Hou", "Yanjie Zhao", "Yue Liu"],
       "year": 2023,
       "arxiv_id": "2308.10620",
       "relevance": "Comprehensive survey of LLMs in SE, providing context for the evaluation challenges this paper addresses."
     },
     {
       "title": "BatchEval: Towards Human-like Text Evaluation",
       "authors": ["Peiwen Yuan", "Shaoxiong Feng", "Yiwei Li"],
       "year": 2024,
       "relevance": "Key inference strategy evaluated in this study; proposes multi-round scoring with diversified batching."
     },
     {
       "title": "Scaling LLM Test-Time Compute Optimally can be More Effective than Scaling Model Parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Relevant to the finding that inference-time scaling strategies provide only marginal improvement for SE evaluation."
     },
     {
       "title": "Split and Merge: Aligning Position Biases in Large Language Model based Evaluators",
       "authors": ["Zongjie Li", "Chaozheng Wang", "Pingchuan Ma"],
       "year": 2023,
       "arxiv_id": "2310.01432",
       "relevance": "Addresses position bias in LLM-as-a-judge, relevant to the inconsistency findings in pairwise comparison (RQ3)."
     }
   ]
 }