ai-research-survey

scan-v5.json (21910B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "Evaluating Judges as Evaluators: The JETTS Benchmark of LLM-as-Judges as Test-Time Scaling Evaluators",
     "authors": [
       "Yilun Zhou",
       "Austin Xu",
       "Peifeng Wang",
       "Caiming Xiong",
       "Shafiq Joty"
     ],
     "year": 2025,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2504.15253",
     "doi": "10.48550/arXiv.2504.15253"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All three abstract claims (judges competitive with ORMs in reranking, worse than PRMs in beam search, critiques ineffective) are directly supported by experimental results in Sections 4.2–4.4 and the leaderboard in Figure 1.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims judge-specific finetuning 'seems to primarily boost instruction-following abilities, sometimes at the cost of other capabilities' (Sec 4.2) based on observational regression, not controlled training experiments. The study design is evaluative, not causal.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are scoped to 'current crop of judge models' and three tested domains. The conclusion frames findings as limitations of current judges, not universal statements about all possible LLM-judges.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternatives for major findings. For example, poor critique performance could stem from the generator's inability to act on critiques rather than critique quality itself — this distinction is not explored.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Normalized helpfulness is defined as a function of actual task performance (accuracy, pass@1, win rate). Measurement granularity directly matches the claims made.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Section 5 is 'Conclusion and Future Work' and the Impact Statement is a single generic sentence claiming no societal consequences need highlighting.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The one specific threat identified (oracle over-estimation in beam search due to answer vs. solution accuracy) appears in Appendix B.2, not as a systematic validity analysis. No threats regarding benchmark dataset selection or judge sampling are discussed.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what results do not generalize to. Findings are presented without explicit scope boundaries such as model size ranges or task types where conclusions would not hold.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section is present. All authors are from Salesforce AI Research but no independent funding source is mentioned.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All five authors are clearly identified as being from Salesforce AI Research in the paper header and correspondence information.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are Salesforce employees and SFR-Judge (a Salesforce-developed model family with 8B/12B/70B variants) is one of the primary models benchmarked. The authors evaluate their own product without disclosing this conflict.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. The Impact Statement does not address financial interests, patents, equity, or consulting relationships.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "LLM-judges are defined as 'models trained to generate evaluations and critiques in natural language'; test-time scaling is defined; normalized helpfulness is formally defined with equations; all three benchmark tasks are precisely specified.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly claims to propose 'the first systematic benchmark of LLM-judges for model's test-time scaling' in the introduction, with clear description of what JETTS adds over existing judge benchmarks.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 systematically compares JETTS against RewardBench, ProcessBench, PPE, JudgeBench, and critique evaluation benchmarks, explaining how each differs and what gap JETTS fills.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues that RewardBench uses responses from different generators allowing judges to exploit stylistic differences, while JETTS uses responses from the same generator. Figure 2 empirically demonstrates that RewardBench performance diverges from JETTS for smaller models, providing evidence that JETTS measures a more fundamental capability.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "No difficulty distribution analysis is provided. Datasets range from easy (GSM8k) to hard (MATH Level 5) but there is no systematic characterization of item difficulty distribution within the benchmark.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "No explicit ceiling or floor effect analysis is conducted. The normalized helpfulness metric partially compensates, but the paper does not analyze whether certain datasets fail to discriminate between judges.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline is included. Comparisons are made against greedy decoding, random selection, oracle (best-of-N), majority vote, and reward models only.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "The normalized helpfulness metric (h = (p_judge - p_greedy) / (p_oracle - p_greedy)) is formally defined with explicit justification for each component. The effective improvement ratio for refinement is similarly justified to ensure it beats both greedy and reranking baselines simultaneously.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "The benchmark uses pre-existing datasets (GSM8k, MATH, HumanEval+, etc.) with no contamination resistance built in. No canary strings, temporal splits, or dynamic generation mechanisms are employed.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether JETTS will remain discriminative as models improve, whether the chosen judge and generator models will become obsolete, or any plans for benchmark updates.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The paper identifies that oracle accuracy in beam search is a severe over-estimate due to final-answer vs. solution-validity divergence (App B.2), and discusses how GPT-4o stochasticity in CHAMP evaluation is mitigated by pre-computing all response evaluations.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full code is released at https://github.com/SalesforceAIResearch/jetts-benchmark, all prompts are provided in the appendix (Figs 15-19), pre-computed model responses are released, and evaluation protocols are described in detail in Appendix A.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 documents all 8 datasets with sizes and evaluation metrics. Appendix A.1 provides detailed evaluation procedures for each dataset. The paper relies on established datasets with existing documentation and thoroughly describes all benchmark-specific components.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "A GitHub link is provided but no explicit license for JETTS is stated in the paper. The licensing terms under which others can use and extend the benchmark are not discussed.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": false,
           "justification": "While a 'Practitioner note' recommends using reranking as a proxy for beam search, there is no explicit statement of what should NOT be concluded from JETTS results or what use cases the benchmark is not designed for.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "LLM-judges are competitive with outcome reward models in response reranking",
       "evidence": "Figure 1 leaderboard shows top judges (SFR 70B: 0.171, SC 70B: 0.177) comparable to Best RM (0.113) in normalized helpfulness",
       "supported": "strong"
     },
     {
       "claim": "LLM-judges consistently underperform process reward models in beam search",
       "evidence": "Figure 9 shows QPRM 7B achieves higher normalized helpfulness than all LLM-judges in beam search for math, including 70B judges",
       "supported": "strong"
     },
     {
       "claim": "Natural language critiques from LLM-judges are ineffective for guiding generator refinement",
       "evidence": "Figure 11 shows all 6 evaluated judges achieve effective improvement ratio below 1.0 across all task categories, indicating critique-based refinement underperforms both greedy and reranking baselines",
       "supported": "strong"
     },
     {
       "claim": "Small judges cannot provide weak-to-strong guidance for large generators",
       "evidence": "Figure 5 regression shows negative normalized helpfulness at judge/generator size ratio ~0.1 for math tasks; 8B judge for 70B generator yields negative helpfulness on average",
       "supported": "strong"
     },
     {
       "claim": "RewardBench performance does not predict judge utility in test-time scaling settings",
       "evidence": "Figure 2 shows small judges perform comparably to large judges on RewardBench but lag significantly on JETTS reranking and beam search, particularly the 8B vs 70B Skywork-Critic comparison",
       "supported": "strong"
     },
     {
       "claim": "Judge effectiveness is domain-dependent: instruction following best, code generation worst",
       "evidence": "Figure 4 shows all judges demonstrate highest helpfulness for instruction following, mixed but mostly positive for math, and mostly negative for code generation — consistent across all judge models",
       "supported": "strong"
     },
     {
       "claim": "Domain-specific prompting does not improve judge performance",
       "evidence": "Figure 23 shows domain-specific prompts for SFR-Judge-8B decrease performance on all benchmarks, though not statistically significantly (all p > 0.05)",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "JETTS reveals that LLM-judges are competitive with outcome reward models for response reranking but are consistently inferior to process reward models in beam search, even when judges are much larger than PRMs. Natural language critiques — a purported key advantage of judges — are currently ineffective at guiding generators toward better responses, with all evaluated judges achieving sub-baseline effective improvement ratios. Judge effectiveness is highly domain-dependent: instruction following works best, math is mixed, and no evaluated judge reliably improves code generation. Existing judge benchmarks like RewardBench fail to predict real-world test-time scaling utility, particularly for distinguishing small from large judges.",
   "red_flags": [
     {
       "flag": "Self-serving benchmark design",
       "detail": "All authors are Salesforce employees and SFR-Judge (a Salesforce-developed judge family with 8B/12B/70B variants) is one of the primary evaluated models. No conflict-of-interest disclosure is made despite this obvious conflict."
     },
     {
       "flag": "Limited critique evaluation scope",
       "detail": "Only 3 of 10 evaluated judges support critique generation, significantly limiting generalizability of the refinement findings. The conclusion that 'all are incapable at this task' rests on a small, potentially unrepresentative sample."
     },
     {
       "flag": "No dedicated limitations section",
       "detail": "The paper has no limitations or threats-to-validity section. The Impact Statement is a single generic sentence claiming no societal consequences need highlighting, inadequate for an ICML paper evaluating AI systems."
     },
     {
       "flag": "Oracle over-estimation buried in appendix",
       "detail": "The paper acknowledges in Appendix B.2 that oracle accuracy in beam search is 'likely a (severe) over-estimate' because it is based on final answer correctness, not solution validity. This materially affects normalized helpfulness interpretation but is not discussed in the main text."
     },
     {
       "flag": "No inter-judge statistical comparisons",
       "detail": "Statistical significance tests compare individual judges against baseline (0) but no pairwise comparisons between judges are reported. Claims about relative judge rankings lack formal statistical backing."
     }
   ],
   "cited_papers": [
     {
       "title": "RewardBench: Evaluating Reward Models for Language Modeling",
       "relevance": "Primary comparison baseline; JETTS is motivated by showing RewardBench inadequately predicts test-time scaling performance"
     },
     {
       "title": "ProcessBench: Identifying Process Errors in Mathematical Reasoning",
       "relevance": "Complementary benchmark for evaluating process reward models; directly relevant to JETTS beam search evaluation"
     },
     {
       "title": "How to Evaluate Reward Models for RLHF (PPE)",
       "relevance": "Related work evaluating reward model efficacy in best-of-N settings; contrasted with JETTS's multi-task approach"
     },
     {
       "title": "JudgeBench: A Benchmark for Evaluating LLM-based Judges",
       "relevance": "Direct predecessor judge benchmark; JETTS extends beyond fixed pairwise test sets to simulate actual test-time compute scenarios"
     },
     {
       "title": "Prometheus 2: An Open Source Language Model Specialized in Evaluating Other Language Models",
       "relevance": "Key evaluated judge model; one of the primary judges across all three JETTS tasks"
     },
     {
       "title": "Scaling LLM Test-Time Compute Optimally Can Be More Effective Than Scaling Model Parameters",
       "relevance": "Core motivation paper establishing test-time compute paradigm with scalar reward models that JETTS extends to LLM-judges"
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "relevance": "Motivates the critique-based refinement task; one of the original papers arguing for natural language feedback loops in agentic settings"
     },
     {
       "title": "Direct Judgement Preference Optimization (SFR-Judge)",
       "relevance": "Describes the SFR-Judge models (Salesforce's judges) that are primary benchmarked models in JETTS"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly answers 'should I use LLM judges or reward models for test-time scaling?' with concrete domain-specific guidance and a practitioner note on using reranking as a proxy metric."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that natural language critiques — a touted advantage of judges — are currently useless for refinement challenges community assumptions; the weak-to-strong failure is also non-obvious."
     },
     "fear_safety": {
       "score": 0,
       "justification": "The paper has no safety or risk implications; it is purely an evaluation methodology paper for LLM judges in test-time compute settings."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild conflict: Salesforce employees benchmark their own SFR-Judge against competitors without disclosure; the finding that RewardBench misleads practitioners may generate some community debate."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Full code released on GitHub with pre-computed responses, allowing practitioners to immediately evaluate new judge models through the JETTS pipeline."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Salesforce AI Research is a recognized lab, ICML 2025 venue, and the evaluated models include well-known systems from multiple organizations."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "40160728",
         "title": "CatLIP: Clip Vision Accuracy with 2.7x Faster Pre-Training on Web-Scale Data",
         "points": 48,
         "comments": 4,
         "url": "https://news.ycombinator.com/item?id=40160728",
         "created_at": "2024-04-25T17:46:04Z"
       },
       {
         "hn_id": "43686458",
         "title": "NPB-Rust: NAS Parallel Benchmarks in Rust",
         "points": 6,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43686458",
         "created_at": "2025-04-14T21:21:43Z"
       },
       {
         "hn_id": "41517885",
         "title": "Towards Large Language Models as Copilots for Theorem Proving in Lean",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41517885",
         "created_at": "2024-09-12T05:34:47Z"
       },
       {
         "hn_id": "40086186",
         "title": "Toward Self-Improvement of LLMs via Imagination, Searching, and Criticizing",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40086186",
         "created_at": "2024-04-19T12:51:23Z"
       },
       {
         "hn_id": "43781749",
         "title": "A Comprehensive Benchmark for C-to-Safe-Rust Transpilation",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43781749",
         "created_at": "2025-04-24T12:08:53Z"
       },
       {
         "hn_id": "44327775",
         "title": "Approximating Language Model Training Data from Weights",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44327775",
         "created_at": "2025-06-20T13:56:11Z"
       },
       {
         "hn_id": "44086818",
         "title": "Gen2seg: Generative Models Enable Generalizable Instance Segmentation",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44086818",
         "created_at": "2025-05-25T10:20:25Z"
       },
       {
         "hn_id": "40139677",
         "title": "Toward Self-Improvement of LLMs via Imagination, Searching, and Criticizing",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40139677",
         "created_at": "2024-04-24T02:10:20Z"
       },
       {
         "hn_id": "40116933",
         "title": "Toward Self-Improvement of LLMs via Imagination, Searching, and Criticizing",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40116933",
         "created_at": "2024-04-22T18:02:16Z"
       },
       {
         "hn_id": "45349444",
         "title": "Seeing Is Deceiving:Mirror-Based Lidar Spoofing for Autonomous Vehicle Deception",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45349444",
         "created_at": "2025-09-23T16:39:48Z"
       }
     ],
     "top_points": 48,
     "total_points": 71,
     "total_comments": 5
   }
 }