ai-research-survey

scan-v4.json (22421B)

---

 {
   "scan_version": 4,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "Evaluating Judges as Evaluators: The JETTS Benchmark of LLM-as-Judges as Test-Time Scaling Evaluators",
     "authors": [
       "Yilun Zhou",
       "Austin Xu",
       "Peifeng Wang",
       "Caiming Xiong",
       "Shafiq Joty"
     ],
     "year": 2025,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2504.15253",
     "doi": "10.48550/arXiv.2504.15253"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'judges are competitive with outcome reward models in reranking' (Fig. 4, Best RM comparison), 'consistently worse than process reward models in beam search' (Fig. 9, QPRM comparison), 'natural language critiques are currently ineffective' (Fig. 11, all δ(Eff) < 1.0).",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Claims are appropriately hedged as observational: 'judge-specific finetuning seems to primarily boost instruction-following evaluation abilities' (Sec. 4.2), 'This may suggest an intrinsic pliability' (Sec. 4.4). Controlled comparisons (same setup, varying one factor) support the comparative claims adequately.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded by 'current' temporal qualifier and refer to the tested judges. The paper tests 10 judges across 3 domains and 8 generators, providing broad but bounded coverage. The title correctly identifies this as 'The JETTS Benchmark' rather than making universal claims.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Limited systematic discussion of alternatives. For the main finding that critiques are ineffective, the paper identifies failure modes (style over substance) but doesn't consider whether the refinement prompt design (Fig. 18), generator capabilities, or other factors could explain the results. The JETTS vs. RewardBench difficulty gap discussion (Sec. 1) is a notable exception.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Metrics match claims well. Normalized helpfulness (Eq. 1) directly measures what is claimed (judge improvement over greedy baseline, relative to oracle). The paper does not conflate benchmark performance with broader capabilities — claims are about judge performance in specific test-time scaling settings.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 5 ('Conclusion and Future Work') identifies specific limitations with corresponding research directions: the pairwise performance-efficiency dilemma (O(N²)), insufficient CoT reasoning for judgment and critiques, and the need for better reasoning capabilities. This constitutes substantive discussion.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated threats-to-validity discussion. Some specific methodological concerns are noted in passing (CHAMP oracle overestimation in App. B.2, positional bias mitigation in App. B.1), but these are not framed as threats to validity and no systematic analysis of how these could affect conclusions is provided.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit 'what the results do NOT show' statements. The paper does not explicitly state what settings, populations, or claims are excluded from its scope. The practitioner note (Sec. 3.5) offers practical guidance but does not delineate scope boundaries.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section in the paper. All authors are from Salesforce AI Research but no funding sources are mentioned.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All five authors are listed as affiliated with 'Salesforce AI Research' on the first page.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Salesforce AI Research developed SFR-Judge (Wang et al., 2024a), one of the evaluated judge models. SFR-Judge-70B achieves the highest reranking and beam search performance among judges in the leaderboard (Fig. 1), and Salesforce has a commercial interest in demonstrating its judge model's effectiveness.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement appears in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined: 'LLM-as-judge', 'test-time scaling', 'normalized helpfulness' (Eq. 1), and 'effective improvement ratio' (Eq. 3) are all formally defined with mathematical notation.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it proposes 'the first systematic benchmark of LLM-judges for model's test-time scaling' with three specific tasks, and clearly differentiates its contribution from existing benchmarks like RewardBench.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 provides a substantive comparison against RewardBench, PPE, JudgeBench, ProcessBench, and critique evaluation benchmarks, explaining specifically how JETTS differs (simulating actual test-time scenarios vs. fixed pairwise test sets) rather than merely listing related papers.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues that existing benchmarks like RewardBench form pairs from different generators, allowing judges to exploit stylistic shortcuts; JETTS requires comparing responses from the same generator, specifically measuring utility in realistic test-time compute scenarios.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "The benchmark uses multiple datasets of varying difficulty (GSM8k vs. MATH Level 5 vs. CHAMP), but difficulty tiers are not formally characterized or measured; the spread across datasets is implicit rather than systematically analyzed.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "The paper observes that some judges perform below the greedy baseline (negative normalized helpfulness) and that single-rating judges are over-lenient, but does not conduct a formal ceiling/floor analysis or assess whether the benchmark discriminates adequately across the range of evaluated models.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline performance is reported for any of the benchmark tasks; the paper compares only AI models against random and oracle baselines.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Normalized helpfulness (Eq. 1) is defined and justified as measuring improvement beyond greedy with oracle as natural upper bound; the effective improvement ratio (Eq. 3) is justified for refinement where oracle upper bound is unavailable.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "The benchmark relies entirely on existing public datasets (GSM8k, MATH, HumanEval+, etc.) that are widely present in LLM training data; no contamination resistance measures such as temporal splits, canary strings, or dynamic generation are employed.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether the benchmark will be gamed as models improve or provide plans for dataset refreshing; no analysis of how quickly benchmark utility might decay is offered.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly discusses failure modes: oracle accuracy over-estimation for CHAMP math (App. B.2), positional bias (addressed via consistency checks, App. B.1), and qualitative judge critique failure modes (false positives from stylistic focus, false negatives from over-scrutiny, Figs. 27-28).",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Pre-computed model responses are released as part of the benchmark, prompt templates are provided in the appendix (Figs. 15-18), and code is available at the GitHub repository listed in the abstract.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": false,
           "justification": "Table 1 lists the datasets with sizes and metrics, and Appendix A.1 describes evaluation procedures, but no formal data card is provided; documentation for the pre-computed response cache (collection methodology, model generation details) is minimal.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "A GitHub repository URL is provided, but the paper does not state a license for the benchmark materials; it is unclear under what terms others may use or redistribute the pre-computed responses.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": false,
           "justification": "A practitioner note advises using reranking as a proxy for beam search, but the paper does not explicitly specify what cannot be concluded from JETTS results or delineate appropriate vs. inappropriate use cases for the benchmark.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "LLM-judges are competitive with outcome reward models in response reranking",
       "evidence": "SFR-70B (h=0.171) and Skywork-Critic-70B (h=0.177) exceed the best RM baseline (h=0.113) on reranking in Fig. 1",
       "supported": "strong"
     },
     {
       "claim": "LLM-judges are consistently worse than process reward models in beam search",
       "evidence": "Fig. 9 shows Qwen2.5-Math-PRM-7B achieves h=0.195 in beam search while the best judge (SFR-70B) achieves h=0.138; Table 14 confirms QPRM outperforms all judges across most dataset/generator combinations",
       "supported": "strong"
     },
     {
       "claim": "Natural language critiques from LLM-judges are currently ineffective at guiding generators to produce better responses",
       "evidence": "Fig. 11 shows no judge achieves effective improvement ratio >1.0 across all task categories; the seed response (index 0) is most likely to be selected after refinement (Fig. 14)",
       "supported": "strong"
     },
     {
       "claim": "RewardBench fails to reveal capability differences between small and large judges that matter for test-time scaling",
       "evidence": "Fig. 2 shows Skywork-Critic 8B and 70B perform similarly on RewardBench but diverge substantially on JETTS reranking and beam search tasks",
       "supported": "strong"
     },
     {
       "claim": "Larger judge-to-generator size ratio significantly increases helpfulness for math but not for code generation",
       "evidence": "Fig. 5 shows regression coefficient 0.16*** for math vs. 0.00 n.s. for code in reranking; Fig. 8 confirms the same pattern in beam search (Math coef: 0.09***, Code coef: 0.01 n.s.)",
       "supported": "strong"
     },
     {
       "claim": "Judge critiques fail because they focus on stylistic features rather than response correctness",
       "evidence": "Qualitative case study of 100+ critique-response pairs (App. B.3, Figs. 27-28) finds two modes: false positives (judges miss errors, praise formatting) and false negatives (judges over-penalize verbose but correct responses)",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "JETTS reveals that LLM-judges are competitive with outcome reward models (ORMs) in response reranking but substantially lag the 7B Qwen2.5-Math-PRM in step-level beam search, undermining the case for using judges as drop-in RM replacements in test-time compute pipelines. Natural language critiques — considered a defining advantage of LLM-judges over scalar RMs — are currently ineffective: no judge achieves an effective improvement ratio above 1.0 in critique-based refinement across any task category. Existing judge benchmarks like RewardBench mask this capability gap by forming pairs from different generators, allowing stylistic shortcuts; JETTS's within-generator comparison exposes that small judges (8B) provide negligible improvement over a 70B generator even when the 8B model scores similarly to a 70B judge on RewardBench.",
   "red_flags": [
     {
       "flag": "Self-evaluation conflict",
       "detail": "All authors are from Salesforce AI Research, which developed SFR-Judge — one of the evaluated judge families. SFR-Judge appears prominently in all three benchmark tasks with no conflict-of-interest disclosure."
     },
     {
       "flag": "No contamination controls",
       "detail": "The benchmark uses entirely pre-existing public datasets (GSM8k, MATH, HumanEval+, MBPP+, BigCodeBench, AlpacaEval, IFEval) that are widely included in judge model training data, potentially inflating judge performance on familiar problems."
     },
     {
       "flag": "No human baseline",
       "detail": "The benchmark does not establish human performance on any of its tasks, making it impossible to contextualize how close LLM-judges are to human-level evaluation quality."
     },
     {
       "flag": "No dedicated limitations section",
       "detail": "Limitations are folded into the conclusion section without formal enumeration; threats to validity such as training data overlap between judges and test tasks are not addressed."
     },
     {
       "flag": "No license for benchmark artifacts",
       "detail": "Pre-computed responses and benchmark materials are released on GitHub with no stated license, creating ambiguity for downstream use and reproducibility."
     }
   ],
   "cited_papers": [
     {
       "title": "RewardBench: Evaluating Reward Models for Language Modeling",
       "relevance": "Primary comparison benchmark; JETTS explicitly compares against RewardBench to show it reveals judge capability differences that RewardBench masks"
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "relevance": "Foundational LLM-as-judge work that established the paradigm JETTS evaluates in test-time scaling contexts"
     },
     {
       "title": "JudgeBench: A Benchmark for Evaluating LLM-Based Judges",
       "relevance": "Direct related work on judge evaluation; JETTS differentiates itself by simulating actual test-time scaling rather than fixed pairwise tests"
     },
     {
       "title": "ProcessBench: Identifying Process Errors in Mathematical Reasoning",
       "relevance": "Related benchmark for process reward model evaluation; JETTS extends to judge models in beam search"
     },
     {
       "title": "Scaling LLM Test-Time Compute Optimally Can Be More Effective than Scaling Model Parameters",
       "relevance": "Key reference establishing the test-time scaling paradigm that JETTS benchmarks judges against"
     },
     {
       "title": "How to Evaluate Reward Models for RLHF (PPE)",
       "relevance": "Related evaluator benchmark covering Best-of-N settings; JETTS extends to three distinct task types and judge-specific architectures"
     },
     {
       "title": "Prometheus 2: An Open Source Language Model Specialized in Evaluating Other Language Models",
       "relevance": "One of the evaluated judge models; represents the fine-grained evaluation with natural language feedback approach"
     },
     {
       "title": "Direct Judgement Preference Optimization (SFR-Judge)",
       "relevance": "Salesforce-developed judge model central to the evaluation; training methodology relevant to understanding judge capabilities"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly actionable for ML practitioners choosing between LLM-judges and reward models for inference-time compute; benchmark and pre-computed responses are released for immediate use."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that natural language critiques — widely touted as judges' key advantage — are currently useless for refinement challenges the prevailing optimism about LLM-as-judge capabilities."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Minor safety relevance: unreliable evaluators in test-time scaling could propagate errors or reinforce incorrect responses at scale, but no explicit safety framing is made."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly challenges the validity of RewardBench as a proxy for real-world judge utility, but the disagreement is framed technically rather than polemically."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Full benchmark with pre-computed responses and code released on GitHub; researchers can immediately reproduce and extend the evaluation with new judge models."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Salesforce AI Research is a recognizable lab, and the paper appears at ICML 2025; evaluates well-known models (Llama, Qwen, DeepSeek) which increases visibility."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "40160728",
         "title": "CatLIP: Clip Vision Accuracy with 2.7x Faster Pre-Training on Web-Scale Data",
         "points": 48,
         "comments": 4,
         "url": "https://news.ycombinator.com/item?id=40160728",
         "created_at": "2024-04-25T17:46:04Z"
       },
       {
         "hn_id": "43686458",
         "title": "NPB-Rust: NAS Parallel Benchmarks in Rust",
         "points": 6,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43686458",
         "created_at": "2025-04-14T21:21:43Z"
       },
       {
         "hn_id": "41517885",
         "title": "Towards Large Language Models as Copilots for Theorem Proving in Lean",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41517885",
         "created_at": "2024-09-12T05:34:47Z"
       },
       {
         "hn_id": "40086186",
         "title": "Toward Self-Improvement of LLMs via Imagination, Searching, and Criticizing",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40086186",
         "created_at": "2024-04-19T12:51:23Z"
       },
       {
         "hn_id": "43781749",
         "title": "A Comprehensive Benchmark for C-to-Safe-Rust Transpilation",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43781749",
         "created_at": "2025-04-24T12:08:53Z"
       },
       {
         "hn_id": "44327775",
         "title": "Approximating Language Model Training Data from Weights",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44327775",
         "created_at": "2025-06-20T13:56:11Z"
       },
       {
         "hn_id": "44086818",
         "title": "Gen2seg: Generative Models Enable Generalizable Instance Segmentation",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44086818",
         "created_at": "2025-05-25T10:20:25Z"
       },
       {
         "hn_id": "40139677",
         "title": "Toward Self-Improvement of LLMs via Imagination, Searching, and Criticizing",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40139677",
         "created_at": "2024-04-24T02:10:20Z"
       },
       {
         "hn_id": "40116933",
         "title": "Toward Self-Improvement of LLMs via Imagination, Searching, and Criticizing",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40116933",
         "created_at": "2024-04-22T18:02:16Z"
       },
       {
         "hn_id": "45349444",
         "title": "Seeing Is Deceiving:Mirror-Based Lidar Spoofing for Autonomous Vehicle Deception",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45349444",
         "created_at": "2025-09-23T16:39:48Z"
       }
     ],
     "top_points": 48,
     "total_points": 71,
     "total_comments": 5
   }
 }