ai-research-survey

scan.json (29584B)

---

 {
   "paper": {
     "title": "Grading Scale Impact on LLM-as-a-Judge: Human-LLM Alignment Is Highest on 0-5 Grading Scale",
     "authors": [
       "Weiyue Li",
       "Minda Zhao",
       "Weixuan Dong",
       "Jiahui Cai",
       "Yuze Wei",
       "Michael Pocress",
       "Yi Li",
       "Wanyan Yuan",
       "Xiaoyue Wang",
       "Ruoyu Hou",
       "Kaiyuan Lou",
       "Wenqi Zeng",
       "Yutong Yang",
       "Yilun Du",
       "Mengyu Wang"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.03444",
     "doi": "10.48550/arXiv.2601.03444"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository, GitHub link, or archive URL is provided anywhere in the paper or appendices."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The underlying benchmarks (STS-B, MT-Bench, ToxiGen, etc.) are public, but the paper's primary data contribution — the collected human and LLM ratings across scales — is not released or linked."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency files, or library versions are provided. The paper does not describe the software environment used for computing ICC values or running LLM APIs."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, scripts, or step-by-step guide are provided for replicating the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All ICC and nMAE values in Tables 1-6 are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 0-5 is the best scale and 0-10 is consistently the weakest based on comparing raw ICC/nMAE values across tables, but no significance tests (e.g., bootstrap test on ICC differences) are performed to determine whether these differences are statistically meaningful."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw ICC and nMAE values but does not report formal effect sizes. Differences between scales (e.g., ICC of 0.853 vs 0.805) are stated without quantifying effect magnitude using standard measures like Cohen's d."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 150 items (25 per benchmark), 12 human raters, and 6 LLMs is not justified. No power analysis or sample size rationale is given for any of these design parameters."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results are single-run ICC/nMAE point estimates. The temperature ablation (Table 6) shows results at four temperatures but only for two models and without reporting variance across temperatures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study compares three grading scales (0-5, 0-10, 0-100) against each other and also contrasts their ICC approach against Pearson correlation (used by Lee et al. 2025), as discussed in Section 5.4 and Figure 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The comparison with Lee et al. (2025) Pearson correlation approach and the use of contemporary LLM models (GPT-4o, Gemini-2.5-flash, DeepSeek-v3.2) makes the baselines current."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 presents a temperature ablation study testing robustness across T ∈ {0.1, 0.4, 0.7, 1.0} for two models, verifying that scale ordering is not an artifact of decoding strategy."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses both ICC (absolute agreement) and nMAE (normalized mean absolute error) throughout, and explicitly discusses why both are informative (Section 5.4, Figure 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The entire study centers on human evaluation — 12 graduate student annotators provide ratings across all items and scales in a fully crossed design (Sections 3.2, 4.2)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No train/test split is employed. The same 150 items are used for all analyses with no separation for tuning vs. evaluation, though the study does not involve model training or tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-benchmark breakdowns are provided in Tables 1, 3, 4a, and Appendix Tables 14-15, showing reliability and agreement for each of the six benchmarks separately."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 provides error analysis with representative poorly-aligned cases (Table 16) and well-aligned cases (Table 17), analyzing why specific examples lead to high/low agreement."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that on MT-Bench, 0-10 actually outperforms 0-5 for human-LLM agreement (Table 4a: ICC 0.570 vs 0.517), contradicting the overall finding. The paper also reports that LLM panel reliability drops sharply on subjective benchmarks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about 0-5 maximizing agreement (Table 2), scale-dependent consistency drops on subjective benchmarks (Table 1), and gender subgroup differences (Table 5) are all supported by the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims scale choice 'shifts' human-LLM agreement. The fully crossed within-subjects design (same raters, same items, different scales, randomized order) with session spacing of ~1 week adequately supports this causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Human-LLM Alignment Is Highest on 0-5 Grading Scale' presents a general finding, but the evidence comes from only 12 graduate student raters, 6 LLMs, and 6 benchmarks. While the limitations section acknowledges the graduate student sample, the title and abstract do not bound the claim to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for why 0-5 yields better alignment. Possible explanations (e.g., cognitive load, anchor point effects from psychometric literature, model tokenization artifacts) are not explored. The temperature ablation rules out one confounder but no others."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully distinguishes ICC from Pearson correlation (Figure 2, Section 5.4) and explains why ICC measures absolute agreement while correlation only measures relative ranking. The paper's claims match the granularity of its measurements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists 'gpt-4o' and 'gemini-2.5-flash' without snapshot dates or API version identifiers. Per schema rules, marketing names like 'GPT-4o' without snapshot dates do not count as specified versions. Other models (Llama-3.3-70B-Instruct, Qwen3-32B, Mistral-7B-Instruct-v0.3) have more specific identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates for all six benchmarks are provided in Appendix C (Tables 8-13), including system messages and user instructions. Placeholders are for data items from public benchmarks and scale bounds, so prompts are fully reconstructable."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states: 'For all LLM judges, we set the default temperature to 1. We also use non-thinking mode for all models.' However, top-p, max tokens, and other API parameters are not reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. LLMs are prompted directly with a single judging template and return a score."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '150 items total, with 25 items sampled from each benchmark' but does not describe the sampling method (random, stratified, etc.). The linear mapping to [0,1] for inter-scale ICC is documented, but item selection criteria are not."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present with substantive discussion of the graduate student sample, task difficulty, and generalizability constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section identifies specific threats: 'The human raters are all graduate students and thus do not represent the broader populations,' 'some items...are inherently difficult even for humans,' and 'uncertainty can lower agreement with LLMs and may also make ICC estimates sensitive to benchmark composition.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations discuss factors that constrain generality but do not explicitly state what the results do NOT show or what claims the authors are NOT making. No equivalent of 'what the evidence does not show' statements."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw rating data (individual human and LLM scores per item per scale) is not released or made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 and Appendix D describe the data collection: Label Studio platform, randomized item/scale order, separate sessions spaced ~1 week apart, fractional scoring allowed, 12 annotators with informed consent."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A states: 'Human annotators are recruited on a voluntary basis from graduate students across multiple institutions. Participation is entirely voluntary, and all annotators provide informed consent.'"
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw ratings to final ICC values has unexplained steps. How were the 25 items per benchmark sampled? Were any annotations excluded? How were LLM outputs parsed? These steps are not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, grants, or sponsorship is mentioned anywhere in the paper. There is no acknowledgments section addressing funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Harvard University, CMU, Stanford University, UC San Diego. The authors are not affiliated with the LLM companies whose models they evaluate."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funder is disclosed, so independence cannot be assessed. The absence of funding disclosure makes this NO."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the six LLM judges. The models (GPT-4o, Gemini-2.5-flash, etc.) likely trained on data containing these benchmarks, but this is not discussed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmarks used (STS-B from 2017, MT-Bench from 2023, etc.) predate the models and are widely available online. The potential that models memorized benchmark items or associated human ratings is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All six benchmarks were published before the models' training periods (STS-B 2017, SummEval 2021, MT-Bench 2023, etc.). The risk that models have seen these items and associated evaluations during training is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned. No link to OSF, AsPredicted, or any registry."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite involving human annotators exposed to potentially sensitive content (ToxiGen). The ethical considerations section discusses risks but does not mention IRB review."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Demographics reported: 12 graduate students, 6 female and 6 male, from multiple institutions (Harvard, CMU, Stanford, UC San Diego). Education level and gender distribution are specified."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "The only stated criterion is 'graduate students across multiple institutions' with voluntary participation. No formal inclusion/exclusion criteria, screening process, or eligibility requirements are described."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes randomization: 'Human raters complete three rating blocks (one per scale) in randomized order, and within each block, item order is shuffled to reduce anchoring and fatigue effects.'"
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Blinding to the grading scale is not feasible — raters must know what scale they are using. This criterion does not apply."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition or dropout information is reported. The paper states a 'fully crossed item-rater design' implying all 12 raters completed everything, but this is not explicitly confirmed."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token consumption, or per-item latency is reported for the LLM judging experiments, despite querying six models across thousands of items on three scales."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or hardware specifications are stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No random seed sensitivity analysis is performed. The temperature ablation (Section 5.3) explores sensitivity to a decoding parameter but not random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The main results appear to be from a single run per model per scale. The number of runs is not explicitly stated. Only the temperature ablation section states multiple configurations."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Temperature is set to 1 as 'default' without justifying this choice or reporting alternatives tried (the ablation is post-hoc for two models only)."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All three scale configurations are reported and compared (Tables 2, 4a, 4b). The 'best' scale (0-5) emerges from comparing all configurations rather than selecting a single best result."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparisons across 3 scales × 6 benchmarks × 6 models × 2 genders without any correction for multiple comparisons (Bonferroni, Holm, etc.)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors do not discuss potential bias from designing the evaluation framework and interpreting the results. No independent evaluation or acknowledgment of author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "The study measures agreement between raters, not model performance as a function of compute. Compute differences are not relevant to the research question."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses six established benchmarks without questioning whether they adequately test what the paper claims to evaluate — whether scale choice affects 'alignment.' No discussion of whether these benchmarks are representative of real LLM-as-a-judge use cases."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. LLMs are prompted directly with judging templates."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All six benchmarks predate the evaluated models. The models may have seen benchmark items and associated ratings during training. This temporal concern is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The evaluation prompts provide full task context to LLM judges, which is appropriate, but whether training exposure to these specific items creates implicit leakage is not considered."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Items from the same benchmark may share structural properties, and models may have been trained on these exact items, but independence is not analyzed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is employed. No canary strings, membership inference, or decontamination is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "The 0-5 grading scale yields the strongest pooled human-LLM absolute agreement (ICC = 0.853), while 0-10 is consistently the weakest (ICC = 0.805).",
       "evidence": "Table 2 reports pooled ICC across all benchmarks: 0-5 (0.853), 0-10 (0.805), 0-100 (0.840). nMAE follows the same pattern: 0-5 (0.111), 0-10 (0.122), 0-100 (0.115).",
       "supported": "moderate"
     },
     {
       "claim": "LLM inter-scale self-consistency drops substantially on subjective, open-ended quality benchmarks compared to objective-like benchmarks.",
       "evidence": "Table 1 shows average 3-scale ICC of 0.944 (STS-B) and 0.949 (ToxiGen) vs. 0.740 (MT-Bench) and 0.786 (SummEval).",
       "supported": "strong"
     },
     {
       "claim": "Pooled LLM reliability can mask substantial benchmark heterogeneity — a 'reliability illusion.'",
       "evidence": "Table 3 shows LLM panel ICC on 0-5 scale ranges from 0.573 (SummEval) to 0.969 (STS-B), while the pooled ICC in Table 2 is 0.944.",
       "supported": "strong"
     },
     {
       "claim": "The scale ordering (0-5 best, 0-10 worst) is stable under temperature perturbations.",
       "evidence": "Table 6 shows Llama and Gemini human-LLM ICC across T ∈ {0.1, 0.4, 0.7, 1.0}; the 0-5 scale is highest or competitive at all temperatures for both models.",
       "supported": "moderate"
     },
     {
       "claim": "Both male and female rater groups show highest LLM alignment on the 0-5 scale, with the highest within-group reliability also on 0-5.",
       "evidence": "Table 5 shows Male-LLM ICC: 0.839 (0-5), 0.751 (0-10), 0.832 (0-100); Female-LLM ICC: 0.831 (0-5), 0.805 (0-10), 0.818 (0-100). Within-group ICC is highest on 0-5 for both.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o achieves the strongest individual model human-LLM alignment across all scales.",
       "evidence": "Table 4b shows GPT has the highest ICC at every scale: 0.816 (0-5), 0.760 (0-10), 0.810 (0-100).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Across six benchmarks and six LLM judges, the 0-5 grading scale yields the highest pooled human-LLM absolute agreement (ICC = 0.853) while 0-10 is consistently the weakest. LLM inter-scale self-consistency drops sharply on subjective open-ended tasks (MT-Bench, SummEval) but remains high on objective-like tasks, revealing that pooled reliability statistics can mask substantial benchmark heterogeneity. The scale ordering is robust to temperature perturbations, and both gender subgroups in the annotator pool show highest alignment with LLMs on the 0-5 scale.",
   "red_flags": [
     {
       "flag": "Small annotator pool for broad claims",
       "detail": "Only 12 graduate student annotators (from elite institutions: Harvard, CMU, Stanford, UCSD) are used to establish 'human-LLM alignment.' The title presents findings as general ('Human-LLM Alignment Is Highest on 0-5') despite this narrow, non-representative sample."
     },
     {
       "flag": "No significance testing on core claims",
       "detail": "The central claim that 0-5 is the best scale is based on comparing raw ICC point estimates (0.853 vs 0.805 vs 0.840) without any significance tests or confidence intervals. The observed differences may not be statistically significant given the sample sizes."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "All six benchmarks (STS-B 2017, ToxiGen 2022, SummEval 2021, etc.) were published well before the models' training periods. LLMs may have memorized benchmark items and associated quality judgments during training, which could inflate or distort agreement metrics."
     },
     {
       "flag": "Gender analysis from tiny subgroups",
       "detail": "Gender-stratified analysis (Table 5) is based on only 6 raters per group. Drawing conclusions about gender differences in alignment from n=6 per group is underpowered and risks overinterpretation."
     }
   ],
   "cited_papers": [
     {
       "title": "Judging LLM-as-a-judge with MT-bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "Foundational LLM-as-a-judge framework and MT-Bench benchmark used in this study."
     },
     {
       "title": "G-eval: NLG evaluation using GPT-4 with better human alignment",
       "authors": ["Yang Liu", "Dan Iter", "Yichong Xu"],
       "year": 2023,
       "relevance": "LLM-based NLG evaluation method demonstrating GPT-4 alignment with human judgments."
     },
     {
       "title": "Evaluating the consistency of LLM evaluators",
       "authors": ["Noah Lee", "Jiwoo Hong", "James Thorne"],
       "year": 2025,
       "relevance": "Directly related work on LLM judge consistency, uses Pearson correlations that this paper argues are inferior to ICC."
     },
     {
       "title": "Validating LLM-as-a-judge systems under rating indeterminacy",
       "authors": ["Luke Guerdan", "Solon Barocas", "Kenneth Holstein"],
       "year": 2025,
       "arxiv_id": "2503.05965",
       "relevance": "Examines rating indeterminacy in LLM evaluation, directly relevant to the scale sensitivity this paper studies."
     },
     {
       "title": "ChatEval: Towards better LLM-based evaluators through multi-agent debate",
       "authors": ["Chi-Min Chan", "Weize Chen", "Yusheng Su"],
       "year": 2023,
       "arxiv_id": "2308.07201",
       "relevance": "Multi-agent evaluation framework for improving LLM judge quality."
     },
     {
       "title": "Agent-as-a-judge: Evaluate agents with agents",
       "authors": ["Mingchen Zhuge", "Changsheng Zhao", "Dylan Ashley"],
       "year": 2024,
       "arxiv_id": "2410.10934",
       "relevance": "Agent-based evaluation paradigm extending LLM-as-a-judge to agentic systems."
     },
     {
       "title": "From generation to judgment: Opportunities and challenges of LLM-as-a-judge",
       "authors": ["Dawei Li", "Bohan Jiang", "Liangjie Huang"],
       "year": 2025,
       "relevance": "Comprehensive survey of LLM-as-a-judge covering bias, consistency, and alignment challenges."
     },
     {
       "title": "Are we on the right way to assessing LLM-as-a-judge?",
       "authors": ["Yuanning Feng", "Sinan Wang", "Zhengxiang Cheng"],
       "year": 2025,
       "arxiv_id": "2512.16041",
       "relevance": "Reveals significant reliability problems in top-tier LLM judges under pairwise scoring and transitivity checks."
     },
     {
       "title": "Investigating non-transitivity in LLM-as-a-judge",
       "authors": ["Yi Xu", "Laura Ruis", "Tim Rocktäschel"],
       "year": 2025,
       "arxiv_id": "2502.14074",
       "relevance": "Studies non-transitive preference cycles in LLM judge comparisons, complementary consistency concern."
     },
     {
       "title": "RocketEval: Efficient automated LLM evaluation via grading checklist",
       "authors": ["Tianjun Wei", "Wei Wen", "Ruizhi Qiao"],
       "year": 2025,
       "arxiv_id": "2503.05142",
       "relevance": "Checklist-based LLM evaluation framework aiming to reduce bias and improve robustness."
     },
     {
       "title": "GPTScore: Evaluate as you desire",
       "authors": ["Jinlan Fu", "See Kiong Ng", "Zhengbao Jiang"],
       "year": 2024,
       "relevance": "LLM scoring framework for evaluation that this paper benchmarks against in terms of human alignment methodology."
     },
     {
       "title": "Benchmarking cognitive biases in large language models as evaluators",
       "authors": ["Ryan Koo", "Minhwa Lee", "Vipul Raheja"],
       "year": 2024,
       "relevance": "Studies systematic cognitive biases in LLM evaluators including position and verbosity bias."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners using LLM-as-a-judge can immediately adopt the recommendation to use 0-5 scales over 0-10 or 0-100."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that scale matters is somewhat surprising to those who assumed it was neutral, but not deeply contrarian."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns are raised by this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle; this is a straightforward psychometric study."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, tool, or demo is released."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates well-known models (GPT-4o, Gemini) and comes from recognizable institutions (Harvard, Stanford, CMU) but is not from a major AI lab."
     }
   }
 }