ai-research-survey

scan-v5.json (26019B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Grading Scale Impact on LLM-as-a-Judge: Human-LLM Alignment Is Highest on 0-5 Grading Scale",
     "authors": [
       "Weiyue Li",
       "Minda Zhao",
       "Weixuan Dong",
       "Jiahui Cai",
       "Yuze Wei",
       "Michael Pocress",
       "Yi Li",
       "Wanyan Yuan",
       "Xiaoyue Wang",
       "Ruoyu Hou",
       "Kaiyuan Lou",
       "Wenqi Zeng",
       "Yutong Yang",
       "Yilun Du",
       "Mengyu Wang"
     ],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.03444",
     "doi": "10.48550/arXiv.2601.03444"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims—LLM inconsistency across scales on subjective benchmarks, 0-5 maximizing alignment, 0-10 being weakest, pooled reliability masking heterogeneity, and gender subgroup differences—are backed by Tables 1–5.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The claim that scale choice 'shifts' human-LLM agreement is supported by a fully-crossed design where the same 12 humans and 6 LLMs rate the same 150 items on all three scales, isolating scale as the only varying factor; a temperature ablation (Table 6) further rules out stochasticity.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section explicitly bounds the findings to graduate-student annotators and notes that the best scale can be benchmark-dependent (MT-Bench diverges); the abstract itself states results are 'aggregated over tasks.'",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper rules out decoding stochasticity via the temperature ablation but does not discuss alternative explanations for why 0-5 specifically outperforms 0-10, such as cognitive load differences, clustering effects, or annotator anchoring strategies.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly defines its outcome measures (ICC for absolute agreement, nMAE for absolute deviation) and motivates why ICC is preferred over Pearson correlation, clearly distinguishing what is measured from what is claimed.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears on page 8, separate from the conclusion, with multiple specific points.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats named include: annotator pool limited to graduate students whose educational background and LLM familiarity may skew calibration; underspecified or knowledge-intensive items causing human uncertainty that depresses ICC estimates.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states the conclusion is aggregate and that 'the best scale can be benchmark-dependent (e.g., MT-Bench)', and that results should not generalize beyond their annotator population.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section or grant disclosure appears anywhere in the paper text.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations (Harvard University, CMU, Stanford University, UC San Diego) are stated on the title page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funder is disclosed, so independence cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure appears in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "ICC is defined via two-way random-effects ANOVA with explicit formulas (Equations 1–3); nMAE is formally defined (Equations 4–5); 'human-LLM alignment' is operationalized as ICC(A,1) on a 2-column matrix of human-consensus vs. LLM-ensemble scores.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The introduction explicitly states the paper fills a gap by 'investigating inter-scale consistency and its impact on human-LLM agreement' and is 'the first to assess how the choice of score scale affects human-LLM alignment.'",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The related work section positions the paper against Lee et al. (2025)'s Pearson-correlation approach, explains why ICC is superior, and maps contributions onto multiple existing threads (judge bias, consistency, human alignment).",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No code repository or release is mentioned anywhere in the paper.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "The six benchmarks used are public, but the primary data contribution—12 annotators' human ratings across all scales—is not stated to be released.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements file, Docker image, or software environment specification is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are included; the appendices provide prompts and annotation instructions but not a runnable pipeline.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All ICC and nMAE values in Tables 1–6 are point estimates with no confidence intervals or bootstrapped error ranges reported.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are applied to scale comparisons (e.g., no test that 0-5 ICC=0.853 is significantly higher than 0-10 ICC=0.805).",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "ICC values are reported numerically (e.g., 0-5: 0.853 vs 0-10: 0.805 in Table 2), providing interpretable magnitude of differences in a standardized metric.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No justification is given for 12 annotators or 150 items; no power analysis is presented for the ICC comparisons.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Per-benchmark breakdowns show variability across benchmarks but no within-cell variance or confidence bounds are reported for any ICC estimate.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "All three scales (0-5, 0-10, 0-100) serve as mutual comparators; all six models are compared against each other and against the human panel.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "LLM judges include GPT-4o, Gemini-2.5-flash, Llama-3.3-70B, Qwen3-32B, DeepSeek-v3.2, and Mistral-7B—all contemporary models.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.3 presents an explicit temperature ablation (T∈{0.1,0.4,0.7,1.0}) for Llama and Gemini, showing scale ordering is stable across decoding strategies.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Both ICC (absolute agreement) and nMAE (mean absolute deviation normalized by scale range) are reported throughout Tables 2–5.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "12 graduate student annotators rated all 150 items on all three scales in a fully-crossed design, with separate sessions per scale spaced at least one week apart.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": false,
           "answer": false,
           "justification": "This is a reliability/agreement study, not a prediction task; held-out test sets are not applicable.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down per benchmark (Tables 3, 4a), per model (Table 4b), and per gender subgroup (Table 5).",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.4 and Appendix G (Tables 16–17) present representative poorly-aligned and well-aligned cases with explanations of why disagreements occur.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "0-10 is reported as consistently weakest; LLM judges perform poorly on MT-Bench and SummEval; MT-Bench shows scale-dependent behavior that does not uniformly favor 0-5.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Exact model versions are stated: gpt-4o, gemini-2.5-flash (Comanici et al. 2025), Llama-3.3-70B-Instruct, Qwen3-32B, DeepSeek-v3.2, Mistral-7B-Instruct-v0.3.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full prompts for all six benchmarks are provided in Appendices C–D (Tables 8–13), including both system and user messages with all placeholders identified.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Temperature=1 and non-thinking mode are stated for all LLM judges; fractional scoring is allowed for both humans and LLMs.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used; LLMs are called directly with single prompts.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Sampling procedure (25 items per benchmark, 150 total), randomization of item and scale order, and session spacing (≥1 week between scale blocks) are documented.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Neither the human annotation scores nor the LLM-generated scores are stated to be publicly released.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Data collection is described in detail: Label Studio platform, voluntary recruitment, informed consent, randomized item order, separate sessions per scale spaced ≥1 week apart.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A describes voluntary recruitment from graduate students at multiple institutions, fully informed consent, no identifying information collected, and annotator awareness of potentially sensitive content.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The pipeline from raw scores to ICC computation via two-way random-effects ANOVA is described with equations; the LLM querying pipeline (standardized templates, single numeric output) is described, though no code is released.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "This study evaluates calibration and agreement properties of LLM judges, not downstream benchmark performance; training cutoff contamination is not applicable.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "NA — the study does not evaluate model capabilities on novel benchmarks; the benchmarks are used as stimulus items for rating behavior.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "NA — potential prior exposure to benchmark items is not the focus; the study assesses rating calibration, not knowledge recall.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": true,
           "answer": false,
           "justification": "No pre-registration is mentioned anywhere in the paper.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": true,
           "answer": false,
           "justification": "Appendix A mentions informed consent and voluntary participation but does not mention IRB approval or institutional ethics review.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": true,
           "answer": false,
           "justification": "Only gender (6 female, 6 male) and academic status (graduate students) are reported; no age, race, institution, or domain expertise breakdown is provided.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": true,
           "answer": false,
           "justification": "No explicit inclusion or exclusion criteria for annotator selection are stated beyond 'graduate students across multiple institutions.'",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": true,
           "answer": true,
           "justification": "Item order within each scale block is shuffled, and scale block order is randomized across annotators to reduce anchoring and fatigue effects.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": true,
           "answer": false,
           "justification": "No blinding of annotators to scale conditions or study hypotheses is mentioned.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": true,
           "answer": false,
           "justification": "All 12 annotators appear to have completed the study, but dropout rates and any exclusions are not explicitly reported.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference cost, API call counts, or latency measurements are reported for any of the six LLM judges.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Total computational budget (GPU hours, API cost) is not stated anywhere in the paper.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "The 0-5 grading scale yields the highest human-LLM alignment aggregated over six benchmarks (ICC=0.853, nMAE=0.111), while 0-10 is consistently the weakest (ICC=0.805, nMAE=0.122).",
       "evidence": "Table 2 reports pooled human-LLM ICC and nMAE across all three scales; the ordering is consistent across both metrics.",
       "supported": "strong"
     },
     {
       "claim": "LLM inter-scale agreement degrades substantially on subjective, open-ended benchmarks (MT-Bench avg ICC ~0.740, SummEval ~0.786) but remains high on objective-like benchmarks (STS-B ~0.944, ToxiGen ~0.949).",
       "evidence": "Table 1 reports per-benchmark, per-model inter-scale ICC across all three scales and pairwise comparisons.",
       "supported": "strong"
     },
     {
       "claim": "Pooled LLM panel reliability appears excellent (~0.944–0.950) but is dominated by objective benchmarks, masking much lower reliability on subjective tasks (SummEval LLM ICC=0.573, MT-Bench=0.632 at 0-5 scale).",
       "evidence": "Table 3 shows per-benchmark internal reliability on the 0-5 scale with a delta column (ICCHuman − ICCLLM) reaching 0.357 for SummEval.",
       "supported": "strong"
     },
     {
       "claim": "Scale-dependent calibration differences dominate over decoding stochasticity: the 0-5 > 0-100 > 0-10 ordering is stable across temperatures T∈{0.1,0.4,0.7,1.0}.",
       "evidence": "Table 6 reports per-model per-scale human-LLM ICC at four temperatures for Llama and Gemini.",
       "supported": "strong"
     },
     {
       "claim": "Female raters align more with LLMs on the 0-10 scale whereas male raters align more on 0-5 and 0-100.",
       "evidence": "Table 5 shows ICCFemale-LLM=0.805 on 0-10 vs ICCMale-LLM=0.751; ICCMale-LLM=0.839 on 0-5 vs ICCFemale-LLM=0.831.",
       "supported": "moderate"
     },
     {
       "claim": "GPT achieves the strongest pooled human alignment across all scales (0-5: 0.816), followed by Gemini (0.782); Mistral is weakest (0.596).",
       "evidence": "Table 4b reports model-wise human-LLM ICC at each scale; GPT leads on all three scales.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "Grading scale is a significant but underexplored parameter in LLM-as-a-judge protocols: the 0-5 scale achieves the highest absolute human-LLM agreement (ICC=0.853) while 0-10 is consistently weakest, even when within-group panel reliability is near-perfect on all scales. LLM inter-scale consistency degrades substantially on subjective open-ended benchmarks (MT-Bench, SummEval) but remains high on objective-like benchmarks, creating a 'reliability illusion' where pooled metrics conceal benchmark-specific failures. Gender subgroup analysis reveals that female and male annotators differ in which scale best aligns with LLM judgments, underscoring that scale effects are not uniform across annotator subpopulations. These findings motivate treating scale design as a controllable, diagnostically important component of LLM evaluation protocols.",
   "red_flags": [
     {
       "flag": "Underpowered gender subgroup analysis",
       "detail": "Gender comparisons are drawn from 6 raters per group; ICC estimates from 6 raters are highly unstable and no confidence intervals or significance tests accompany the reported differences."
     },
     {
       "flag": "No statistical significance tests on scale comparisons",
       "detail": "The central claim that 0-5 > 0-100 > 0-10 is never tested for significance; ICC differences of ~0.05 between scales could be within sampling noise given 150 items and 12 raters."
     },
     {
       "flag": "No confidence intervals on ICC estimates",
       "detail": "All ICC values are point estimates; ICC CIs can be wide with small rater pools and are standard in psychometric practice (Koo & Li 2016 cited by this paper explicitly recommends them)."
     },
     {
       "flag": "Human annotations not released",
       "detail": "The primary empirical contribution—human ratings across all scales and benchmarks—is not made available, preventing independent replication."
     },
     {
       "flag": "No IRB disclosure",
       "detail": "The study involves human participants rating potentially offensive content (ToxiGen, MoralChoice) but does not report IRB approval."
     },
     {
       "flag": "Graduate student convenience sample",
       "detail": "All 12 annotators are graduate students with LLM familiarity; this pool likely differs systematically from crowd workers or domain experts who are the actual target population for LLM-as-a-judge deployment."
     }
   ],
   "cited_papers": [
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "relevance": "Introduces MT-Bench and the LLM-as-a-judge paradigm; core benchmark and methodology used directly in this work."
     },
     {
       "title": "Evaluating the Consistency of LLM Evaluators",
       "relevance": "Prior work on LLM evaluator consistency under prompt perturbation; directly compared against—uses Pearson correlation where this paper argues for ICC."
     },
     {
       "title": "A Guideline of Selecting and Reporting Intraclass Correlation Coefficients for Reliability Research",
       "relevance": "Methodological foundation for ICC use; the paper's primary statistical tool is ICC(A,1) and ICC(A,k) as specified here."
     },
     {
       "title": "G-Eval: NLG Evaluation Using GPT-4 with Better Human Alignment",
       "relevance": "Representative LLM-as-a-judge paper that this work positions against; uses fixed scales without inter-scale analysis."
     },
     {
       "title": "Validating LLM-as-a-Judge Systems Under Rating Indeterminacy",
       "relevance": "Concurrent work on LLM judge reliability under discrete Likert scales; cited as motivation for examining scale effects."
     },
     {
       "title": "Are We on the Right Way to Assessing LLM-as-a-Judge?",
       "relevance": "Shows ~25% inconsistency in pairwise LLM judge scoring; motivates rigorous measurement of judge reliability."
     },
     {
       "title": "Intraclass Correlations: Uses in Assessing Rater Reliability",
       "relevance": "Original ICC formulation (Shrout & Fleiss 1979); foundational statistical reference for the paper's methodology."
     },
     {
       "title": "LLM Evaluators Recognize and Favor Their Own Generations",
       "relevance": "Documents self-preference bias in LLM judges; part of the systematic bias landscape this work situates itself within."
     },
     {
       "title": "SummEval: Re-Evaluating Summarization Evaluation",
       "relevance": "One of the six core benchmarks used; the subjective benchmark where human-LLM agreement is weakest in this study."
     },
     {
       "title": "ChatEval: Towards Better LLM-Based Evaluators Through Multi-Agent Debate",
       "relevance": "Multi-agent judge framework alternative to single-judge scoring; cited as a competing approach this work complements."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly actionable: anyone running LLM-as-a-judge evaluations should use a 0-5 scale rather than the common 0-10 scale based on this finding."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that finer-grained 0-10 and 0-100 scales yield worse human alignment than 0-5 is counterintuitive given psychometric intuitions about granularity."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or risk concerns raised; this is a measurement methodology paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Challenges the tacit convention of using 0-10 scales in prominent benchmarks like MT-Bench, but framed constructively rather than confrontationally."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Practitioners could immediately reproduce the comparison by running any LLM on public benchmarks with the provided prompts across all three scales."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from Harvard, CMU, Stanford, and UCSD lend some institutional credibility, but no famous lab product or named system is involved."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }