ai-research-survey

scan.json (32302B)

---

 {
   "paper": {
     "title": "Evaluating Judges as Evaluators: The JETTS Benchmark of LLM-as-Judges as Test-Time Scaling Evaluators",
     "authors": [
       "Yilun Zhou",
       "Austin Xu",
       "Peifeng Wang",
       "Caiming Xiong",
       "Shafiq Joty"
     ],
     "year": 2025,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2504.15253",
     "doi": "10.48550/arXiv.2504.15253"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository link provided in the paper header: https://github.com/SalesforceAIResearch/jetts-benchmark."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 3.1 states 'we pre-compute the model responses and release them as part of the benchmark wherever possible.' All datasets used (GSM8k, MATH, CHAMP, HumanEval+, MBPP+, BigCodeBench, AlpacaEval, IFEval) are publicly available benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (1 A100 40GB for ≤12B models, 8xA100 40GB for ≥8x7B models) and vLLM is named as the inference framework (App. A.2), but no requirements.txt, Dockerfile, or detailed dependency specifications are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions appear in the paper text. While a GitHub repository is linked, the paper itself does not contain a 'Reproducing Results' section or equivalent."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 3-10 report point estimates only. Significance markers (*, **, ***) are shown on figures but no confidence intervals or ± notation appears. Figure 7 shows min/max ranges from tied responses but these are not statistical error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 4.1: 'we perform statistical analyses of reported quantity differing from the baseline value (0 for reranking and beam-search, and 1 for refinement) using a two-sided one-sample t-test and indicate the significance as \"*\" for p ≤0.05, \"**\" for p ≤0.01, \"***\" for p ≤0.001.'"
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Normalized helpfulness (Eq. 1) contextualizes judge performance relative to greedy (baseline) and oracle (upper bound), providing magnitude context. Effective improvement ratio (Eq. 3) similarly reports relative magnitudes. Full normalized values are in Tables 11-15."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Dataset sizes are listed in Table 1 (ranging from 164 for HumanEval+ to 1,324 for MATH) but no justification is given for why these sizes are adequate, nor is a power analysis performed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Responses are pre-computed and fixed to minimize randomness, but this means only single-run results are reported. No standard deviation, IQR, or variance across repeated experiments is provided. The min/max ranges in Fig. 7 reflect tied-response ambiguity, not experimental variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines included: random reranking, greedy decoding, majority vote (for math), vanilla Llama-3.1-8B-Instruct with judge prompt, three outcome reward models (OffsetBias-RM, Skywork-Reward-8B, Skywork-Reward-27B), and a process reward model (Qwen2.5-Math-PRM-7B)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baseline reward models are recent: Skywork-Reward-v0.2, Qwen2.5-Math-PRM, OffsetBias-RM. Generator models include recent releases (Qwen-2.5, Llama-3.1). All judge models are from 2024."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablation-style analyses: pairwise vs. single-rating protocols (Sec. 4.2), Likert vs. additive rubrics, domain-specific vs. generic prompts (Fig. 23), lookahead vs. non-lookahead beam search (Sec. 4.3), judge size ratio effects (Fig. 5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: normalized helpfulness (Eq. 1) for reranking and beam search, effective improvement ratio (Eq. 3) for refinement, plus dataset-specific metrics (accuracy via Math-Verify, Pass@1, win rate, prompt-level strict accuracy per Table 1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "App. B.3: 'We inspect over 100 refinements and find two major issues.' The authors manually inspected critique-response pairs and identified two failure modes (false positives from style focus, false negatives from over-scrutinizing style), shown in Figs. 27-28."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmark test sets are used (GSM8k, MATH, HumanEval+, MBPP+, BigCodeBench, AlpacaEval, IFEval, CHAMP). Judge models are not tuned on these benchmarks during this study."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns provided: per-dataset (Fig. 3, Tables 3-10), per-judge (Fig. 4), per-generator (Figs. 6, 13), per-task-category (math/code/instruction following), and per-protocol (pairwise/Likert/additive). Full raw tables in Appendix."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases discussed extensively: negative helpfulness on code tasks (Fig. 4), judges worse than greedy on multiple benchmarks, detailed qualitative failure analysis of critiques in App. B.3 with specific examples (Figs. 27-28)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Major negative findings reported: judges consistently worse than PRMs in beam search, critique-based refinement never achieves effective improvement >1.0 (Fig. 11), domain-specific prompts hurt performance (Fig. 23), code evaluation performance is negative for most judges, single-rating protocols produce over-lenient results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'judges are competitive with outcome reward models in reranking' (Fig. 4, Best RM comparison), 'consistently worse than process reward models in beam search' (Fig. 9, QPRM comparison), 'natural language critiques are currently ineffective' (Fig. 11, all δ(Eff) < 1.0)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Claims are appropriately hedged as observational: 'judge-specific finetuning seems to primarily boost instruction-following evaluation abilities' (Sec. 4.2), 'This may suggest an intrinsic pliability' (Sec. 4.4). Controlled comparisons (same setup, varying one factor) support the comparative claims adequately."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded by 'current' temporal qualifier and refer to the tested judges. The paper tests 10 judges across 3 domains and 8 generators, providing broad but bounded coverage. The title correctly identifies this as 'The JETTS Benchmark' rather than making universal claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Limited systematic discussion of alternatives. For the main finding that critiques are ineffective, the paper identifies failure modes (style over substance) but doesn't consider whether the refinement prompt design (Fig. 18), generator capabilities, or other factors could explain the results. The JETTS vs. RewardBench difficulty gap discussion (Sec. 1) is a notable exception."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Metrics match claims well. Normalized helpfulness (Eq. 1) directly measures what is claimed (judge improvement over greedy baseline, relative to oracle). The paper does not conflate benchmark performance with broader capabilities — claims are about judge performance in specific test-time scaling settings."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with versions and sizes provided in Table 2: e.g., 'Llama-3.1-8B-Instruct', 'Qwen-2.5-72B-Instruct', 'SFR-Judge-70B', 'Skywork-Critic-8B'. For open-source models, these names uniquely identify checkpoints. GPT-4o is used only for CHAMP grading, not as primary subject."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts provided in Appendix: pairwise evaluation (Fig. 15), Likert rating (Fig. 16), additive rating (Fig. 17), refinement prompt (Fig. 18), and domain-specific evaluation criteria (Fig. 19). The partial response note for beam search is also included."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Sampling parameters reported: temperature 1.0, top-p 0.95 (Sec. 3.2). Beam search: (10,2) configuration with depth limit 10 (Sec. 3.3). Refinement: (1,9) setup (Sec. 3.4). Integer rating scale 1-5 specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The benchmark evaluates judge models directly on reranking, beam search, and refinement tasks without an agentic framework."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Response generation procedure documented: greedy + 9 sampled responses per query (Sec. 3.2). Evaluation procedures detailed per dataset in App. A.1. Pre-computation and storage of all responses and GPT-4o evaluations to ensure consistency. Pairwise consistency check with position swapping described (App. B.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 5 ('Conclusion and Future Work') identifies specific limitations with corresponding research directions: the pairwise performance-efficiency dilemma (O(N²)), insufficient CoT reasoning for judgment and critiques, and the need for better reasoning capabilities. This constitutes substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated threats-to-validity discussion. Some specific methodological concerns are noted in passing (CHAMP oracle overestimation in App. B.2, positional bias mitigation in App. B.1), but these are not framed as threats to validity and no systematic analysis of how these could affect conclusions is provided."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit 'what the results do NOT show' statements. The paper does not explicitly state what settings, populations, or claims are excluded from its scope. The practitioner note (Sec. 3.5) offers practical guidance but does not delineate scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Pre-computed model responses are released as part of the benchmark (Sec. 3.1: 'we pre-compute the model responses and release them as part of the benchmark wherever possible'). This allows independent verification of judge evaluations."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection described: 8 public benchmarks listed with sizes and metrics (Table 1), response generation procedure (greedy + 9 sampled with temperature 1.0, top-p 0.95), evaluation methods per dataset detailed in App. A.1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks (GSM8k, MATH, CHAMP, HumanEval+, MBPP+, BigCodeBench, AlpacaEval, IFEval)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: query selection from datasets → response generation (greedy + sampled) → judge evaluation (pairwise or single-rating) → metric computation (normalized helpfulness, improvement ratio). Pre-computation of GPT-4o and GPT-4 Turbo evaluations for consistent grading across judges."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section in the paper. All authors are from Salesforce AI Research but no funding sources are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All five authors are listed as affiliated with 'Salesforce AI Research' on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Salesforce AI Research developed SFR-Judge (Wang et al., 2024a), one of the evaluated judge models. SFR-Judge-70B achieves the highest reranking and beam search performance among judges in the leaderboard (Fig. 1), and Salesforce has a commercial interest in demonstrating its judge model's effectiveness."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the judge models or generator models. The judges may have seen benchmark solutions (GSM8k, MATH, HumanEval all published 2021) during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether judge or generator models' training data overlaps with the benchmark datasets. Judges could have memorized correct solutions, affecting their ability to identify correct responses."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No contamination analysis despite using well-known public benchmarks (GSM8k from 2021, MATH from 2021, HumanEval from 2021) with models trained well after these benchmarks were published. The contamination risk for both generators and judges is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated using benchmark datasets and model-generated responses."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Computational complexity is discussed qualitatively (O(N²) for pairwise vs. O(N) for single-rating, Sec. 5), and hardware is mentioned (App. A.2), but no actual inference costs, latency measurements, or per-example timing are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware specifications mentioned (1 A100 40GB for ≤12B models, 8xA100 40GB for ≥8x7B in App. A.2), but total GPU hours, wall-clock time, or total compute budget for the experiments are not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. Responses are pre-computed once to 'minimize the effect of randomness' (Sec. 3.1), but no analysis of how different random seeds would affect the sampled responses or resulting judge evaluations."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of samples per query (10) and beam search configuration (10,2) are stated, but the paper does not explicitly state how many times the full experiment was run. Pre-computed responses imply a single run, but this is not stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. Temperature (1.0), top-p (0.95), and beam parameters (10,2) appear chosen without justification or search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 3.5: 'If a judge supports multiple protocols (e.g., both single rating and pairwise comparisons in reranking), we report the maximum aggregate performance among all protocols.' The selection criterion is transparent and all raw data for all protocols is provided in the appendix tables."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many t-tests are performed across 8 datasets, 10 judges, and up to 8 generators without any correction for multiple comparisons (no mention of Bonferroni, Holm, or Benjamini-Hochberg)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SFR-Judge is developed by Salesforce AI Research (the authors' affiliation). SFR-Judge-70B achieves the top leaderboard position among judges. No acknowledgment of potential bias from evaluating their own model alongside others."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses asymptotic complexity (O(N²) vs. O(N)) and notes beam search requires more compute, but does not plot or report performance as a function of actual compute budget. No matched-compute comparisons are provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Sec. 1 explicitly discusses construct validity: 'RewardBench forms pairs of responses from different generators, meaning judges can arrive at the right outcome via stylistic factors, like formatting. JETTS, however, requires judges to compare responses sampled from the same generator, limiting stylistic factors.' Fig. 2 compares JETTS vs. RewardBench performance."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Judges are evaluated directly on their output quality without agentic scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Benchmarks like GSM8k (2021), MATH (2021), and HumanEval (2021) predate all evaluated models' training. Judge models trained on data including these benchmark solutions could have an unfair advantage in identifying correct responses."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information to the judges. For example, formatting patterns in correct vs. incorrect responses could serve as features."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between judge/generator training data and benchmark test sets. Some judge models were likely trained on data containing benchmark problems and solutions."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods applied. No canary strings, membership inference, or decontamination analysis performed."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM-judges are competitive with outcome reward models in response reranking but consistently worse than process reward models in beam search.",
       "evidence": "Fig. 1 leaderboard: best judges (SFR-70B at 0.171, SC-70B at 0.177) outperform best RM (0.113) in reranking but lag QPRM (0.195) in beam search. Fig. 9 shows QPRM-7B substantially outperforms all judges in math beam search despite being smaller.",
       "supported": "strong"
     },
     {
       "claim": "Natural language critiques from judges are currently ineffective in guiding generators toward better responses via iterative refinement.",
       "evidence": "Fig. 11: No judge achieves effective improvement ratio δ(Eff) > 1.0 for any task category. Instruction following experiences up to 10% degradation. Sec. 4.4 shows the final selected response is often the seed (index 0), and δ(G)_rand ≈ 1 across datasets.",
       "supported": "strong"
     },
     {
       "claim": "Pairwise comparison protocol outperforms single-instance rating protocols for judge reranking.",
       "evidence": "Fig. 3 shows pairwise protocol achieves higher normalized helpfulness than Likert or additive single-rating on most datasets. Sec. 4.2 notes this comes at O(N²) cost vs. O(N). Fig. 25 shows single-rating judges are over-lenient, rating many responses equally high.",
       "supported": "strong"
     },
     {
       "claim": "Weak judges (8B) cannot reliably improve strong generators (70B+) in reasoning-intensive tasks like math and code.",
       "evidence": "Fig. 5: At judge/generator size ratio ~0.1, normalized helpfulness is negative on average for math. Linear regression coefficient for math is 0.16 (p<0.001), showing statistically significant relationship between size ratio and helpfulness. Code shows no improvement at any size ratio.",
       "supported": "moderate"
     },
     {
       "claim": "No judge can reliably outperform majority-vote aggregation on math datasets.",
       "evidence": "Fig. 6: Only SFR-70B and SC-70B offer improvement over majority-vote on GSM8k, and only for weak generators. On MATH, only SFR-70B beats majority-vote for small-to-medium generators.",
       "supported": "strong"
     },
     {
       "claim": "Judge-specific finetuning primarily boosts instruction-following evaluation abilities, sometimes at the cost of code evaluation.",
       "evidence": "Fig. 4: Vanilla Llama-3.1-8B performs at greedy level across tasks. Finetuned 7B/8B judges 'outperform it significantly on instruction-following and moderately on math, but performance degrades, sometimes significantly, on coding.' Domain-specific prompts do not mitigate this (Fig. 23).",
       "supported": "moderate"
     },
     {
       "claim": "JETTS reveals a scale-dependent judging ability gap not visible in RewardBench.",
       "evidence": "Fig. 2: Small judges (8B) perform comparably to large judges (70B) on RewardBench but substantially worse on JETTS. Skywork-Critic-8B vs 70B differ by ~4% on RewardBench but the 8B yields 'substantively lower improvements over the greedy response' on JETTS.",
       "supported": "strong"
     },
     {
       "claim": "Judge critiques over-focus on stylistic features rather than response correctness.",
       "evidence": "App. B.3 qualitative study of 100+ refinements: Fig. 27 shows judge giving high score to mathematically incorrect response for being 'easy to follow'. Fig. 28 shows judge criticizing correct response for having 'more steps than necessary', leading to unnecessary refinement.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The JETTS benchmark reveals that LLM-judges are competitive with outcome reward models for response reranking (especially in instruction following) but consistently lag process reward models in step-level beam search. Natural language critiques, a unique advantage of judges over scalar reward models, are currently ineffective for guiding generator refinement — no evaluated judge achieves an effective improvement ratio above 1.0. The pairwise evaluation protocol substantially outperforms single-rating protocols but at O(N²) computational cost. Critically, weak judges cannot improve strong generators on reasoning tasks, and the scale gap between judges that JETTS reveals is masked by simpler benchmarks like RewardBench.",
   "red_flags": [
     {
       "flag": "Self-evaluation bias",
       "detail": "SFR-Judge is developed by Salesforce AI Research (the authors' affiliation). SFR-Judge-70B achieves the highest reranking and beam search scores among judges on the leaderboard (Fig. 1). This conflict is not acknowledged or discussed anywhere in the paper."
     },
     {
       "flag": "No contamination analysis",
       "detail": "All benchmarks (GSM8k, MATH, HumanEval, MBPP, published 2021) predate the training of all evaluated models. Judges that have memorized correct solutions during training could artificially outperform in reranking tasks. No contamination check is performed."
     },
     {
       "flag": "No multiple comparison correction",
       "detail": "Hundreds of t-tests are performed across 8 datasets × 10 judges × up to 8 generators. No family-wise error rate correction is applied, inflating the likelihood of false significant findings."
     },
     {
       "flag": "Single-run results",
       "detail": "Responses are pre-computed once and fixed. While this ensures consistency across judge evaluations, it means results depend entirely on one set of sampled responses. No analysis of sensitivity to different response samples is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "RewardBench: Evaluating Reward Models for Language Modeling",
       "authors": ["Nathan Lambert", "Valentina Pyatkin", "Jacob Morrison"],
       "year": 2024,
       "arxiv_id": "2403.13787",
       "relevance": "Primary comparison benchmark for evaluating reward models and LLM-judges; JETTS positions itself against RewardBench's limitations."
     },
     {
       "title": "Scaling LLM Test-Time Compute Optimally Can Be More Effective Than Scaling Model Parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Key prior work on test-time compute scaling with fixed reward models; JETTS extends this by systematically evaluating LLM-judges in the same settings."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "Foundational work on LLM-as-judge paradigm and MT-Bench evaluation framework."
     },
     {
       "title": "Prometheus 2: An Open Source Language Model Specialized in Evaluating Other Language Models",
       "authors": ["Seungone Kim", "Juyoung Suk", "Shayne Longpre"],
       "year": 2024,
       "arxiv_id": "2405.01535",
       "relevance": "One of the primary evaluated judge models; represents open-source judge model development with flexible evaluation criteria."
     },
     {
       "title": "Self-Taught Evaluators",
       "authors": ["Tianlu Wang", "Ilia Kulikov", "Olga Golovneva"],
       "year": 2024,
       "arxiv_id": "2408.02666",
       "relevance": "Evaluated judge model using iterative DPO self-teaching for pairwise evaluation; represents alternative training methodology for judges."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath"],
       "year": 2024,
       "relevance": "Key prior work on using natural language critiques for iterative response refinement in agentic settings."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Developed HumanEval+ and MBPP+ benchmarks with improved testing rigor, used as code generation evaluation datasets in JETTS."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Code generation benchmark with diverse function calls used in JETTS evaluation; represents more complex code generation assessment."
     },
     {
       "title": "Large Language Monkeys: Scaling Inference Compute with Repeated Sampling",
       "authors": ["Bradley Brown", "Jordan Juravsky", "Ryan Ehrlich"],
       "year": 2024,
       "arxiv_id": "2407.21787",
       "relevance": "Studies scaling test-time compute through repeated sampling for code and math tasks, complementary to JETTS's judge-focused evaluation."
     },
     {
       "title": "Let's Verify Step by Step",
       "authors": ["Hunter Lightman", "Vineet Kosaraju", "Yuri Burda"],
       "year": 2023,
       "arxiv_id": "2305.20050",
       "relevance": "Foundational work on process reward models for step-level verification in math reasoning; PRMs shown to outperform judges in JETTS beam search."
     },
     {
       "title": "JudgeBench: A Benchmark for Evaluating LLM-Based Judges",
       "authors": ["Sijun Tan", "Siyuan Zhuang", "Kyle Montgomery"],
       "year": 2024,
       "arxiv_id": "2410.12784",
       "relevance": "Identifies shortcomings in RewardBench's reasoning samples and proposes harder pairwise evaluation for judges; complementary judge evaluation benchmark."
     },
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2024,
       "relevance": "Key prior work on self-refinement using model feedback; JETTS tests whether external judge critiques can achieve what self-refinement promises."
     }
   ]
 }