ai-research-survey

scan-v5.json (27556B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Foundational Automatic Evaluators: Scaling Multi-Task Generative Evaluator Training for Reasoning-Centric Domains",
     "authors": [
       "Austin Xu",
       "Xuan-Phi Nguyen",
       "Yilun Zhou",
       "Chien-Sheng Wu",
       "Caiming Xiong",
       "Shafiq Joty"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2510.17793",
     "doi": "10.48550/arXiv.2510.17793"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (FARE-8B challenging larger evaluators, FARE-20B surpassing 70B+ models, near-oracle MATH reranking, 14.1% RL training gain, 65% code evaluation improvement) are all backed by Tables 1-3 and Figures 3-5.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about training components are supported by ablation studies in Table 6, which systematically vary direct judgment data proportion, curriculum learning, and CoT retention strategy.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly scopes claims to reasoning-centric domains in the title and throughout, and reports per-benchmark performance rather than sweeping generalizations.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for FARE's strong performance — whether results stem primarily from data scale, base model quality, training method, or domain coverage is not systematically disentangled.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes benchmark evaluation (static benchmarks for evaluator quality) from downstream real-world performance (RL training, inference-time reranking), with appropriate metrics for each.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Brief future work mentions appear in Appendix B.2 but no limitations or threats-to-validity section exists.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed, such as benchmark saturation, base model contamination, or limited evaluator generalization outside tested reasoning domains.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper focuses on reasoning-centric domains but does not explicitly state what its results do NOT show (e.g., no claims about non-English, long-form creative, or multilingual evaluation settings).",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure is present. All authors are Salesforce AI Research employees but no external funding or grant acknowledgment appears in the paper.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are clearly identified as Salesforce AI Research affiliates on the title page with contact emails.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Salesforce employees train and evaluate their own FARE models; there is no independent evaluation by parties without a stake in the outcome.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosures, or financial interest declarations appear anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "The paper formally defines automatic evaluator (AE), input/output spaces, and all five evaluation tasks (pairwise, step-level, reference-based verification, reference-free verification, single rating) with mathematical notation in Section 2.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions are enumerated in the introduction: multi-task dataset curation, scalable RS-SFT training recipe, and the FARE family of evaluators with rigorous evaluation.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 and Appendix A thoroughly situate FARE relative to prompted evaluators, SFT/DPO-trained evaluators, RL-trained evaluators, and earlier foundational evaluators, explaining key differences from STE, EvalPlanner, CompassJudger, and J1.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No code release is mentioned anywhere in the paper. The training framework (OpenRLHF, verl) is referenced but no repository link for the FARE training pipeline is provided.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "The 2.5M curated training samples are not released. Evaluation uses public benchmarks, but the novel training dataset (including synthetic data and rubrics) is proprietary.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "While OpenRLHF and verl frameworks are named and hyperparameters listed, no requirements file, Dockerfile, or full dependency specification is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Appendix B.2 provides training hyperparameters but without code, training data, or step-by-step instructions, the work cannot be reproduced.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results in Tables 1-4 and Figures 3-5 are single point estimates with no confidence intervals or error bars reported.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are applied to any comparative claims; performance differences are stated as absolute point improvements without any testing of whether they exceed chance.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Absolute point differences are consistently reported in context (e.g., FARE-8B beats J1-8B by 13.71 points on JudgeBench, 14.1% relative gain over string-matching verifiers), providing effect size context.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The 2.5M training sample size is motivated by the scaling hypothesis from prior work but no formal sample size justification or power analysis is provided.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "All benchmark evaluations are single runs with no variance, standard deviation, or inter-run variability reported across any experiment.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Extensive baselines are included: RISE-Judge, EvalPlanner, J1, RM-R1, CompassJudger, Atla Selene, SFR-Judge, Skywork-Critic, StepWiser, and frontier models like GPT-4o, GPT-5, and gpt-oss-120B.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines include very recent 2025 RL-trained models (J1, RM-R1, StepWiser) and frontier models (GPT-5, gpt-oss-120B), all contemporary at time of publication.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Table 6 ablates proportion of direct judgment data (30-70%), continuous curriculum vs. random shuffling, and CoT retention strategy for the 20B model, quantifying each component's impact on pairwise and step-level benchmarks.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "The paper uses consistent accuracy for pairwise benchmarks, F1 for ProcessBench, Pearson correlation for single-rating tasks, and accuracy for VerifyBench, across 7 core benchmarks and 3 downstream settings.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "The paper trains and evaluates automated evaluators on automated benchmarks; human evaluation of FARE outputs is not conducted and is clearly not relevant to this benchmarking paradigm.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "All evaluation is on held-out test benchmarks (JudgeBench, ProcessBench, VerifyBench, etc.) separate from training data, with explicit N-gram decontamination applied.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "ProcessBench results are broken down by difficulty (GSM8K, MATH, OlympiadBench, OmniMATH); CodingJudgeBench by task type; JETTS provides per-generator and per-benchmark breakdowns in Table 10.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section D.6 explicitly notes FARE-8B fails to improve larger generators on harder benchmarks in JETTS; D.2 shows removing CoT from FARE-20B degrades most benchmark scores; Table 4 shows SC hurts MBPP+.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Negative results include: self-consistency degrades FARE performance on MBPP+, removing CoT from FARE-20B reduces most benchmark scores, and FARE-8B cannot universally improve generator performance in reranking.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Base models are specifically identified as Qwen3-8B-Base and gpt-oss-20B with arXiv citations; all 12 generator models for synthetic data are enumerated by name and model family in Appendix B.1.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Appendix E.1 provides full verbatim prompts for pairwise evaluation, direct judgment pairwise, step-level evaluation, and reference-based verification, with all placeholder variables identified.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Training hyperparameters are reported throughout: batch size 128, learning rate 1e-6, rollout batch sizes 50K/250K, K=4 rollout samples at temperature 0.9, and KL coefficient 0.001 for GRPO experiments.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "This paper trains evaluator models without agentic scaffolding; no agentic framework is used in the experimental setup.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Appendix B.1 describes N-gram decontamination, hand-crafted rubric creation per dataset, programmatic error injection details, and the generate-then-grade procedure with temperature sampling specifics.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "The 2.5M training samples are not released; only Table 5 listing source datasets is provided, making independent verification of the curated dataset impossible.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.1 and Appendix B.1 describe both existing data collection (sources, rubric creation) and synthetic data generation (programmatic error injection and generate-then-grade) in substantial detail with Table 5 enumerating all 24 source datasets.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; all data comes from existing public datasets and automated synthesis pipelines.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from seed datasets through rubric creation, response generation (12 generators), correctness grading, N-gram decontamination, and final dataset composition is documented across Section 3.1 and Appendix B.1.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training cutoffs for the base models (Qwen3-8B-Base, gpt-oss-20B) are not stated, making it unclear whether benchmark examples were available during base model pre-training.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix B.1 explicitly states they applied N-gram matching decontamination following Guha et al. (2025) to remove fine-tuning training samples overlapping with evaluation benchmarks.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly addresses potential benchmark contamination through N-gram matching decontamination of training sets and focuses on modern (2024+) datasets to reduce temporal overlap.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; pre-registration is not applicable.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; IRB/ethics approval is not applicable.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "The paper discusses efficiency as a design goal and compares model sizes/active parameters, but reports no specific inference latency, throughput, or cost numbers.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training details (batch size, rollout batch size, steps) are provided but total GPU-hours or compute budget for training FARE-8B or FARE-20B is not disclosed.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "FARE-8B outperforms RL-trained evaluators of comparable or larger size on JudgeBench",
       "evidence": "Table 1 shows FARE-8B scores 55.71 on JudgeBench vs J1-8B (42.00) and RM-R1-14B (46.86), a 13.71 and 8.85 point margin respectively",
       "supported": "strong"
     },
     {
       "claim": "FARE-20B sets a new standard for open-source evaluators, surpassing specialized 70B+ models",
       "evidence": "Table 1 shows FARE-20B (64.29 JudgeBench, 74.4 PPE) outperforming EvalPlanner-70B (56.60, 70.2) and J1-70B (60.00, 72.8) despite 3.5x fewer total and ~20x fewer active parameters",
       "supported": "strong"
     },
     {
       "claim": "FARE-20B achieves near-oracle inference-time reranking performance on MATH",
       "evidence": "Figure 3 shows FARE-20B approaching the oracle green line on MATH across multiple generators, outperforming SFR-Judge-70B by 14 points and Skywork-Critic-70B by 21 points on Llama-3.1-8B generator",
       "supported": "strong"
     },
     {
       "claim": "Using FARE-20B as verifiers in GRPO training improves downstream model performance by 14.1% over string-matching verifiers",
       "evidence": "Figure 4 shows Qwen2.5-7B-Base trained with FARE-20B verifier reaches 45.2 vs 39.6 (string matching); the 14.1% figure is relative improvement, single run without variance",
       "supported": "moderate"
     },
     {
       "claim": "Continual finetuning of FARE-20B for code with only 15K samples (FARE-20B-Code) outperforms gpt-oss-120B on average",
       "evidence": "Figure 5 shows FARE-20B-Code average consistent accuracy exceeds gpt-oss-120B across three CodingJudgeBench tasks, with 10.48 point gain on test-case quality over FARE-20B",
       "supported": "moderate"
     },
     {
       "claim": "Large-scale RS-SFT without RL is competitive with RL-trained specialized evaluators",
       "evidence": "FARE-8B and FARE-20B trained with rejection sampling SFT outperform RL-trained models (J1, RM-R1, StepWiser) on most benchmarks in Tables 1-3",
       "supported": "strong"
     },
     {
       "claim": "Positional robustness in pairwise evaluation emerges as a function of training data scale",
       "evidence": "Figure 6 shows pairwise consistency increasing monotonically from ~65% to ~80% as training samples increase from 0 to 2.5M for both Qwen3 and Qwen2.5 initializations",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "FARE demonstrates that scaling training data to 2.5M multi-task, multi-domain samples with iterative rejection sampling SFT achieves state-of-the-art performance for generative evaluators without computationally expensive RL training. FARE-8B at 8B parameters matches or exceeds specialized RL-trained evaluators at 14B+ parameters on reasoning benchmarks, while FARE-20B with 3.6B active parameters outperforms dense 70B+ specialized judges across 7 benchmarks. In downstream applications, FARE-20B achieves near-oracle best-of-10 reranking on MATH and yields 14.1% relative improvement over string-matching verifiers in GRPO RL training. An additional finding is that positional robustness emerges naturally with data scale, suggesting data-driven training can mitigate common evaluator biases without targeted interventions.",
   "red_flags": [
     {
       "flag": "No statistical testing",
       "detail": "All comparative claims are made on single-run point estimates without confidence intervals, error bars, or significance tests, making it impossible to determine if performance differences are reliable or within noise."
     },
     {
       "flag": "No code or training data release",
       "detail": "Neither the training pipeline code nor the 2.5M curated training samples are released, making reproduction effectively impossible despite the hyperparameter details provided."
     },
     {
       "flag": "Self-evaluation only",
       "detail": "All evaluations are conducted by the Salesforce team that developed FARE with no independent evaluation by external parties."
     },
     {
       "flag": "No compute budget disclosed",
       "detail": "Total GPU-hours or compute cost for training FARE-8B and FARE-20B is not reported, preventing assessment of practical reproducibility or cost-effectiveness."
     },
     {
       "flag": "Base model contamination unaddressed",
       "detail": "Training cutoffs for base models (Qwen3-8B-Base, gpt-oss-20B) are not stated; these models' pretraining data may overlap with evaluation benchmarks in ways the fine-tuning-level N-gram decontamination cannot address."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations section; scope boundaries regarding language, domain coverage, model scale, and benchmark generalization are not explicitly stated."
     }
   ],
   "cited_papers": [
     {
       "title": "Foundational Autoraters: Taming Large Language Models for Better Automatic Evaluation",
       "relevance": "Direct precursor introducing the foundational evaluator training paradigm; FARE extends this with larger data scale and iterative training"
     },
     {
       "title": "Direct Judgement Preference Optimization",
       "relevance": "Related multi-task foundational evaluator using direct judgment data; key methodological comparison and baseline"
     },
     {
       "title": "Self-Taught Evaluators",
       "relevance": "Related iterative SFT approach for training evaluators; contrasted with FARE in terms of data scale, task coverage, and training stability"
     },
     {
       "title": "J1: Incentivizing Thinking in LLM-as-a-Judge via Reinforcement Learning",
       "relevance": "Key RL-trained evaluator baseline that FARE claims to match or outperform despite simpler training methodology"
     },
     {
       "title": "RM-R1: Reward Modeling as Reasoning",
       "relevance": "RL-trained evaluator baseline; FARE-8B outperforms RM-R1-14B on most benchmarks, supporting the data-scaling argument"
     },
     {
       "title": "JudgeBench: A Benchmark for Evaluating LLM-based Judges",
       "relevance": "Primary pairwise reasoning evaluation benchmark used throughout; introduces consistent accuracy metric adopted by this paper"
     },
     {
       "title": "ProcessBench: Identifying Process Errors in Mathematical Reasoning",
       "relevance": "Step-level evaluation benchmark where FARE-20B achieves state-of-the-art performance, matching GPT-5"
     },
     {
       "title": "Evaluating Judges as Evaluators: The JETTS Benchmark of LLM-as-Judges as Test-Time Scaling Evaluators",
       "relevance": "Framework for downstream inference-time scaling evaluation; used to assess FARE as a best-of-N reranker across multiple generators and tasks"
     },
     {
       "title": "General-Reasoner: Advancing LLM Reasoning Across All Domains",
       "relevance": "Provides the WebInstruct-Verified training setup and General-Verifier baseline for GRPO training experiments; FARE-20B verifier is compared against their approach"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "FARE directly addresses high-demand infrastructure needs for scalable evaluators in RL training and inference-time scaling, with demonstrated practical gains in both settings using off-the-shelf training techniques."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the dominant narrative that RL training is necessary for state-of-the-art evaluators, showing simple data scaling with RS-SFT matches or beats RL-trained models at far lower compute cost."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety concerns are raised; the paper is a systems/ML engineering contribution about training better automated evaluators."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly critiques the recent trend toward RL-based evaluator training as unnecessary complexity, but frames this as a finding rather than a confrontational argument."
     },
     "demo_ability": {
       "score": 1,
       "justification": "No model weights release or demo link is provided in the paper text; the models may be available but are not publicized in this preprint."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Salesforce AI Research is a recognized industrial AI lab; the paper benchmarks against and claims to outperform OpenAI's GPT-5 on several evaluation tasks."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45657595",
         "title": "Binary Retrieval-Augmented Reward Mitigates Hallucinations",
         "points": 44,
         "comments": 3,
         "url": "https://news.ycombinator.com/item?id=45657595",
         "created_at": "2025-10-21T16:14:28Z"
       },
       {
         "hn_id": "42984225",
         "title": "Leveraging Multimodal LLM for Inspirational User Interface Search",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42984225",
         "created_at": "2025-02-08T16:52:28Z"
       },
       {
         "hn_id": "45876369",
         "title": "Diagnosing Representation Dynamics in NER Model Extension",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45876369",
         "created_at": "2025-11-10T14:30:09Z"
       }
     ],
     "top_points": 44,
     "total_points": 47,
     "total_comments": 3
   }
 }