ai-research-survey

scan.json (27871B)

---

 {
   "paper": {
     "title": "Foundational Automatic Evaluators: Scaling Multi-Task Generative Evaluator Training for Reasoning-Centric Domains",
     "authors": ["Austin Xu", "Xuan-Phi Nguyen", "Yilun Zhou", "Chien-Sheng Wu", "Caiming Xiong", "Shafiq Joty"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2510.17793",
     "doi": "10.48550/arXiv.2510.17793"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code release link is provided in the paper. The paper mentions using a 'modified version of the OpenRLHF framework' but does not release their modifications or training code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes curating 2.5M training samples from various sources but does not release the curated dataset. Individual source datasets are public, but the assembled training mix is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency lists are provided. The paper mentions using OpenRLHF but does not specify versions or dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While training hyperparameters are listed in §B.2, there are no runnable scripts or commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results tables (Tables 1-3, 4, 7-10) report only point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many claims of the form 'FARE-8B outperforms X by Y points' (e.g., §4.1) based solely on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports absolute point differences with baselines, e.g., 'outperforming RM-R1-14B by 6.57 absolute points on JudgeBench' (§4.1), 'a nearly 11 point absolute gain' (§4.2), providing baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the choice of 2.5M training samples, the number of benchmarks, or benchmark sizes. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experiments. Results appear to be single-run numbers. The K=4 rollout sampling is part of training, not evaluation variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are included across all benchmarks: Tables 1-3 compare against 10+ baselines including RISE-Judge, EvalPlanner, J1, RM-R1, CompassJudger, Atla Selene, and frontier models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are contemporary, including recently released models like J1, RM-R1, StepWiser, GPT-5, and gpt-oss-120B (all 2024-2025 releases)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "§D.1 (Table 6) ablates three components: proportion of direct judgment data (30-70%), continuous curriculum, and CoT handling for gpt-oss. §D.2 ablates critique vs. direct judgment prompting."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: consistent accuracy for pairwise, F1 for ProcessBench, accuracy for VerifyBench, Pearson correlation for single-rating, and downstream task performance for JETTS/GRPO experiments."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of FARE's outputs is included. All evaluation is automated via benchmark scoring. For an evaluator paper, human assessment of evaluation quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation benchmarks (JudgeBench, ProcessBench, VerifyBench, etc.) are external held-out test sets not used during training. The paper mentions N-gram decontamination (§B.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Per-category breakdowns are provided throughout: ProcessBench shows GSM8K/MATH/OlympiadBench/OmniMATH splits (Table 2), JETTS shows per-benchmark results (Table 10), GRPO shows per-benchmark downstream results (Fig. 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§D.2 discusses failure cases where direct judgment prompting degrades FARE-20B performance. Table 4 notes MBPP+ degradation with SC@32. Table 10 shows cases where FARE-8B fails to improve over baseline."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: FARE-20B performance degrades without critique/CoT (Table 7), SC@32 hurts MBPP+ (Table 4), FARE-8B struggles with larger generators on hard benchmarks (§D.6), and Qwen3-8B post-trained is noted as 'over-trained' and difficult to finetune."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'FARE-8B challenges larger specialized RL-trained evaluators' (Tables 1-2), 'FARE-20B surpassing specialized 70B+ evaluators' (Tables 1-2), 'near-oracle performance on MATH' (Fig. 3), '14.1% vs string-matching' (Fig. 4), 'outperforms gpt-oss-20B by 65%' (Fig. 5)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are mostly justified through controlled ablations (Table 6) that isolate individual factors. The GRPO experiment (Fig. 4) controls for training setup, varying only the verifier. Ablation design uses single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Foundational Automatic Evaluators' and the paper frames FARE as general-purpose, but evaluation is concentrated on math/code/reasoning. The 'multi-domain' claim is not well-bounded — chat/safety evaluation is minimal (Table 8 shows only moderate single-rating performance)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations. The gains could be partially due to data contamination, base model selection, or scale effects rather than the RS-SFT recipe, but these are not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures benchmark accuracy as a proxy for 'evaluation quality' but does not discuss this gap. Benchmark performance may not reflect real-world evaluation utility. The JETTS reranking experiment partially addresses this but the distinction is not explicitly acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Base models are identified as 'Qwen3-8B-Base' and 'gpt-oss-20B' but without version/snapshot dates. Baseline models like 'GPT-4o', 'GPT-5' are used without specific API versions. Generator models for synthetic data list model families but not exact versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full evaluation prompts for pairwise, step-level, reference-based verification, and direct judgment are provided in §E.1 with complete text including system and user prompt templates."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "§B.2 reports batch size 128, learning rate 1e-6, rollout batch sizes 50K/250K, K=4 rollouts at temperature 0.9. §C.2 reports GRPO training hyperparameters (rollout batch 1024, group size 8, temp 1.0, KL 0.001, LR 5e-7)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. FARE models are standard generative evaluators without tool use, memory, or multi-step reasoning scaffolds."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "§3.1 describes data curation in detail: existing data sources with hand-crafted rubrics, synthetic data via programmatic error injection and generate-then-grade. Table 5 enumerates all data sources. §B.1 mentions N-gram decontamination."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section in the paper. The conclusion (§5) is brief and does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential issues like benchmark overfitting, selection bias in benchmark choice, or generalization concerns."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what settings or domains FARE has NOT been tested on or where it might fail."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw training data, model outputs, and evaluation predictions are not released. Only aggregate benchmark numbers are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "§3.1 describes data collection in detail: existing data sources are enumerated in Table 5, synthetic data generation uses 12 generators with specific sampling strategies, and rubric creation process is described with an example in §E.2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from existing datasets and synthetic generation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: Table 5 lists all sources, Fig. 2 shows breakdowns by task/domain/approach. §3.1 describes the existing data → synthetic supplement → final mix pipeline. Decontamination is mentioned in §B.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement or acknowledgments section mentions funding sources. All authors are from Salesforce AI Research but no funding disclosure is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as 'Salesforce AI Research' with email addresses. The affiliation is clearly stated on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Salesforce funds this research and has a commercial interest in demonstrating strong AI evaluation capabilities. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Salesforce employees evaluating Salesforce-trained models is an undeclared conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for the base models (Qwen3-8B-Base, gpt-oss-20B) or for the FARE training data collection period."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§B.1 states 'We took efforts to decontaminate our training sets with N-gram matching approaches, following Guha et al. (2025).' This addresses potential overlap between training and evaluation data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While N-gram decontamination is mentioned for FARE's own training data, no analysis is provided for whether the base models (Qwen3, gpt-oss) may have seen the evaluation benchmarks during their pre-training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens-per-evaluation is reported despite the paper emphasizing efficiency as a design goal (§2: 'we seek efficiency'). The SC@32 experiments multiply inference cost 32x without cost discussion."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated. Training uses 2.5M samples with iterative RS-SFT but GPU hours, training time, and total API spend for synthetic data generation are not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results are reported. All benchmark evaluations appear to be single-run. Training uses a single random seed without sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation runs is not stated. For pairwise benchmarks with consistent accuracy, each sample is run twice (swapping order), but no repeated runs for variance estimation."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The ablation study (Table 6) explores a few configurations but does not report total search budget or how the final hyperparameters were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 shows ablation results across configurations and the paper explains selection rationale: 'we choose 40% and 60% to train FARE-8B and FARE-20B' based on peak average performance across pairwise and ProcessBench."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction. The paper compares across many benchmarks and baselines without any correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own FARE models against baselines without acknowledging self-comparison bias. Baselines use officially reported numbers or are run with their own templates (§C.1), but the asymmetry in effort is not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "FARE is trained on 2.5M samples with iterative RS-SFT while baselines may use far less data/compute. No compute-matched comparison is provided. FARE-20B has 3.6B active parameters vs. 70B+ baselines, but training compute is not compared."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 7 benchmarks actually measure meaningful evaluation quality. The paper does not question construct validity of any benchmark."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in FARE evaluation. Models are prompted directly without agentic frameworks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The base models (Qwen3, gpt-oss) may have been trained on data containing benchmark problems, and training data includes datasets from various time periods without temporal analysis."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For example, the reference-based verification task provides a reference answer — no analysis of whether this creates an unfair advantage over baselines that don't use references."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and evaluation data share structural similarities. Training data uses some of the same seed datasets (MATH, MBPP) that appear in evaluation benchmarks, and while N-gram decontamination is applied, distributional overlap is not analyzed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "§B.1 states 'We took efforts to decontaminate our training sets with N-gram matching approaches, following Guha et al. (2025).' This is a concrete leakage prevention method."
       }
     }
   },
   "claims": [
     {
       "claim": "FARE-8B outperforms larger specialized RL-trained evaluators on pairwise benchmarks",
       "evidence": "Table 1: FARE-8B achieves 55.71 on JudgeBench vs. J1-8B (42.00), RM-R1-14B (46.86), and EvalPlanner-8B (30.20). Also outperforms CompassJudger-14B (50.29).",
       "supported": "strong"
     },
     {
       "claim": "FARE-20B surpasses specialized 70B+ evaluators",
       "evidence": "Table 1: FARE-20B (64.29 JudgeBench) vs. EvalPlanner-70B (56.60), J1-70B (60.00). Table 2: FARE-20B (84.4 ProcessBench) vs. Qwen2.5-Math-72B-PRM (78.3).",
       "supported": "strong"
     },
     {
       "claim": "FARE-20B achieves near-oracle reranking performance on MATH",
       "evidence": "Fig. 3: FARE-20B reranking of Llama-3.1-8B on MATH approaches the oracle green line, with 50.83 vs. 53.47 oracle (Table 10).",
       "supported": "strong"
     },
     {
       "claim": "FARE improves downstream RL-trained model performance by up to 14.1% vs. string-matching verifiers",
       "evidence": "Fig. 4: String matching yields 39.6 average, FARE-20B yields 45.2, a relative improvement of (45.2-39.6)/39.6 = 14.1%.",
       "supported": "strong"
     },
     {
       "claim": "Continually-finetuned FARE-Code outperforms gpt-oss-20B by 65% on test-case quality evaluation",
       "evidence": "Fig. 5: FARE-20B-Code achieves 56.55 on test case quality vs. gpt-oss-20B's 34.40, a relative improvement of ~64.4%.",
       "supported": "strong"
     },
     {
       "claim": "Iterative rejection sampling SFT is a stable approach for training evaluators at scale",
       "evidence": "The paper trains on 2.5M samples and shows improving performance across iterations (Fig. 6 shows consistency improving with data scale). Ablation in Table 6 shows stable results across configurations.",
       "supported": "moderate"
     },
     {
       "claim": "Positional robustness emerges with data scale",
       "evidence": "Fig. 6 shows consistency steadily increasing from ~65% to ~80% over 2.5M training samples for both Qwen3-8B and Qwen2.5-7B models.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "FARE, a family of 8B and 20B generative evaluators trained on 2.5M multi-task samples via iterative rejection-sampling SFT, achieves best-in-class performance across 7 evaluation benchmarks spanning pairwise, step-level, and reference-based verification tasks. FARE-20B surpasses specialized 70B+ evaluators despite having ~20x fewer active parameters. In downstream applications, FARE achieves near-oracle reranking on MATH, improves RL-trained model performance by 14.1% over string-matching verifiers, and can be continually finetuned for domain-specific evaluation with only 15K samples.",
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All authors are from Salesforce AI Research. FARE is initialized from gpt-oss-20B (a Salesforce model) and compared favorably against competitors. The paper also introduces gpt-oss-20B/120B as baselines that FARE is shown to improve upon, effectively promoting Salesforce models."
     },
     {
       "flag": "No variance or error bars across any results",
       "detail": "All results across 7 benchmarks and 3 downstream settings are single-point estimates. For a paper making strong comparative claims ('best-in-class', 'sets the new standard'), the absence of any uncertainty quantification is a significant concern."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations discussion whatsoever. For a paper claiming 'foundational' evaluators, failure to discuss where the approach might fail or what settings are out of scope is a notable omission."
     },
     {
       "flag": "No compute cost reported despite emphasizing efficiency",
       "detail": "The paper lists efficiency as a design goal (§2) but never reports training compute, inference latency, or API costs. The 2.5M sample training and 12-generator synthetic data pipeline represent substantial compute, and SC@32 experiments multiply inference cost 32x, all unreported."
     },
     {
       "flag": "Selective benchmark presentation",
       "detail": "FARE-20B underperforms gpt-oss-20B on some benchmarks when prompted without critique/CoT (Table 7: PPE drops from 74.4 to 68.9, RM-Bench from 90.5 to 85.5). The main results use the favorable prompting setup."
     }
   ],
   "cited_papers": [
     {
       "title": "Foundational autoraters: Taming large language models for better automatic evaluation",
       "authors": ["Tu Vu", "Kalpesh Krishna", "Salaheddin Alzubi"],
       "year": 2024,
       "arxiv_id": "2407.10817",
       "relevance": "Prior work on foundational multi-task evaluator training that FARE builds upon."
     },
     {
       "title": "Self-taught evaluators",
       "authors": ["Tianlu Wang", "Ilia Kulikov", "Olga Golovneva"],
       "year": 2024,
       "arxiv_id": "2408.02666",
       "relevance": "Closely related iterative SFT approach for training evaluators, used as baseline."
     },
     {
       "title": "J1: Incentivizing thinking in LLM-as-a-judge via reinforcement learning",
       "authors": ["Chenxi Whitehouse", "Tianlu Wang"],
       "year": 2025,
       "arxiv_id": "2505.10320",
       "relevance": "RL-trained judge model used as primary baseline across pairwise benchmarks."
     },
     {
       "title": "RM-R1: Reward modeling as reasoning",
       "authors": ["Xiusi Chen", "Gaotang Li"],
       "year": 2025,
       "arxiv_id": "2505.02387",
       "relevance": "RL-trained reward model baseline for pairwise evaluation."
     },
     {
       "title": "ProcessBench: Identifying process errors in mathematical reasoning",
       "authors": ["Chujie Zheng", "Zhenru Zhang"],
       "year": 2024,
       "arxiv_id": "2412.06559",
       "relevance": "Key benchmark for step-level evaluation of mathematical reasoning."
     },
     {
       "title": "JudgeBench: A benchmark for evaluating LLM-based judges",
       "authors": ["Sijun Tan", "Siyuan Zhuang"],
       "year": 2024,
       "arxiv_id": "2410.12784",
       "relevance": "Primary pairwise benchmark for evaluating LLM judges in reasoning settings."
     },
     {
       "title": "Evaluating judges as evaluators: The JETTS benchmark",
       "authors": ["Yilun Zhou", "Austin Xu"],
       "year": 2025,
       "arxiv_id": "2504.15253",
       "relevance": "Benchmark for evaluating LLM judges as test-time scaling evaluators, used for downstream evaluation."
     },
     {
       "title": "Direct judgement preference optimization",
       "authors": ["Peifeng Wang", "Austin Xu"],
       "year": 2024,
       "arxiv_id": "2409.14664",
       "relevance": "Training approach for multi-task foundational evaluators at scale."
     },
     {
       "title": "General-reasoner: Advancing LLM reasoning across all domains",
       "authors": ["Xueguang Ma", "Qian Liu"],
       "year": 2025,
       "arxiv_id": "2505.14652",
       "relevance": "Provides the GRPO training setup and verifier baselines used in FARE's downstream evaluation."
     },
     {
       "title": "No free labels: Limitations of LLM-as-a-judge without human grounding",
       "authors": ["Michael Krumdick", "Charles Lovering"],
       "year": 2025,
       "arxiv_id": "2503.05061",
       "relevance": "Documents risks of LLM evaluators generating reference answers, motivating FARE's design choice to avoid this."
     },
     {
       "title": "Inference-time scaling for generalist reward modeling",
       "authors": ["Zijun Liu", "Peiyi Wang"],
       "year": 2025,
       "arxiv_id": "2504.02495",
       "relevance": "DeepSeek-GRM approach for inference-time scaling of evaluators, used as baseline in Table 4."
     },
     {
       "title": "StepWiser: Stepwise generative judges for wiser reasoning",
       "authors": ["Wei Xiong", "Wenting Zhao"],
       "year": 2025,
       "arxiv_id": "2508.19229",
       "relevance": "RL-trained specialized step-level evaluator used as baseline on ProcessBench."
     }
   ]
 }