ai-research-survey

scan-v4.json (32508B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "Foundational Automatic Evaluators: Scaling Multi-Task Generative Evaluator Training for Reasoning-Centric Domains",
     "authors": [
       "Austin Xu",
       "Xuan-Phi Nguyen",
       "Yilun Zhou",
       "Chien-Sheng Wu",
       "Caiming Xiong"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2510.17793",
     "doi": "10.48550/arXiv.2510.17793"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'FARE-8B challenges larger specialized RL-trained evaluators' (Tables 1-2), 'FARE-20B surpassing specialized 70B+ evaluators' (Tables 1-2), 'near-oracle performance on MATH' (Fig. 3), '14.1% vs string-matching' (Fig. 4), 'outperforms gpt-oss-20B by 65%' (Fig. 5).",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are mostly justified through controlled ablations (Table 6) that isolate individual factors. The GRPO experiment (Fig. 4) controls for training setup, varying only the verifier. Ablation design uses single-variable manipulation.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Foundational Automatic Evaluators' and the paper frames FARE as general-purpose, but evaluation is concentrated on math/code/reasoning. The 'multi-domain' claim is not well-bounded — chat/safety evaluation is minimal (Table 8 shows only moderate single-rating performance).",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations. The gains could be partially due to data contamination, base model selection, or scale effects rather than the RS-SFT recipe, but these are not explored.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures benchmark accuracy as a proxy for 'evaluation quality' but does not discuss this gap. Benchmark performance may not reflect real-world evaluation utility. The JETTS reranking experiment partially addresses this but the distinction is not explicitly acknowledged.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section in the paper. The conclusion (§5) is brief and does not discuss limitations.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential issues like benchmark overfitting, selection bias in benchmark choice, or generalization concerns.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what settings or domains FARE has NOT been tested on or where it might fail.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement or acknowledgments section mentions funding sources. All authors are from Salesforce AI Research but no funding disclosure is provided.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as 'Salesforce AI Research' with email addresses. The affiliation is clearly stated on the first page.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Salesforce funds this research and has a commercial interest in demonstrating strong AI evaluation capabilities. The funder is not independent of the outcome.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Salesforce employees evaluating Salesforce-trained models is an undeclared conflict.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 provides a formal mathematical definition of 'automatic evaluator' (πθ: X→Y) and precisely defines all five evaluation tasks (pairwise, step-level, reference-based/free verification, single rating) with input/output structure.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions are explicitly bullet-pointed in Section 1: (1) a 2.5M multi-task training dataset, (2) iterative RS-SFT recipe, and (3) the FARE-8B and FARE-20B model family.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 systematically positions FARE against prompted evaluators, SFT/DPO-based foundational evaluators, RL-trained models, and inference-time scaling approaches, explaining how RS-SFT bridges these paradigms.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No repository URL or code release link is provided in the paper. The paper mentions using a 'modified version of the OpenRLHF framework' but does not release their modifications or training code.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "The paper describes curating 2.5M training samples from various sources but does not release the curated dataset. Individual source datasets are public, but the assembled training mix is not released.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No environment specifications, requirements files, or dependency lists are provided. The paper mentions using OpenRLHF but does not specify versions or dependencies.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided. While training hyperparameters are listed in §B.2, there are no runnable scripts or commands.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results tables (Tables 1-3, 4, 7-10) report only point estimates with no confidence intervals, error bars, or ± notation.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "The paper makes many claims of the form 'FARE-8B outperforms X by Y points' (e.g., §4.1) based solely on comparing numbers without any statistical significance tests.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper consistently reports absolute point differences with baselines, e.g., 'outperforming RM-R1-14B by 6.57 absolute points on JudgeBench' (§4.1), 'a nearly 11 point absolute gain' (§4.2), providing baseline context.",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No justification is given for the choice of 2.5M training samples, the number of benchmarks, or benchmark sizes. No power analysis is discussed.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No variance, standard deviation, or spread measures are reported for any experiments. Results appear to be single-run numbers. The K=4 rollout sampling is part of training, not evaluation variance.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Extensive baselines are included across all benchmarks: Tables 1-3 compare against 10+ baselines including RISE-Judge, EvalPlanner, J1, RM-R1, CompassJudger, Atla Selene, and frontier models.",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines are contemporary, including recently released models like J1, RM-R1, StepWiser, GPT-5, and gpt-oss-120B (all 2024-2025 releases).",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "§D.1 (Table 6) ablates three components: proportion of direct judgment data (30-70%), continuous curriculum, and CoT handling for gpt-oss. §D.2 ablates critique vs. direct judgment prompting.",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Multiple metrics are used: consistent accuracy for pairwise, F1 for ProcessBench, accuracy for VerifyBench, Pearson correlation for single-rating, and downstream task performance for JETTS/GRPO experiments.",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "No human evaluation of FARE's outputs is included. All evaluation is automated via benchmark scoring. For an evaluator paper, human assessment of evaluation quality would be relevant.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "All evaluation benchmarks (JudgeBench, ProcessBench, VerifyBench, etc.) are external held-out test sets not used during training. The paper mentions N-gram decontamination (§B.1).",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Per-category breakdowns are provided throughout: ProcessBench shows GSM8K/MATH/OlympiadBench/OmniMATH splits (Table 2), JETTS shows per-benchmark results (Table 10), GRPO shows per-benchmark downstream results (Fig. 4).",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "§D.2 discusses failure cases where direct judgment prompting degrades FARE-20B performance. Table 4 notes MBPP+ degradation with SC@32. Table 10 shows cases where FARE-8B fails to improve over baseline.",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Several negative results: FARE-20B performance degrades without critique/CoT (Table 7), SC@32 hurts MBPP+ (Table 4), FARE-8B struggles with larger generators on hard benchmarks (§D.6), and Qwen3-8B post-trained is noted as 'over-trained' and difficult to finetune.",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Base models are identified as 'Qwen3-8B-Base' and 'gpt-oss-20B' but without version/snapshot dates. Baseline models like 'GPT-4o', 'GPT-5' are used without specific API versions. Generator models for synthetic data list model families but not exact versions.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full evaluation prompts for pairwise, step-level, reference-based verification, and direct judgment are provided in §E.1 with complete text including system and user prompt templates.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "§B.2 reports batch size 128, learning rate 1e-6, rollout batch sizes 50K/250K, K=4 rollouts at temperature 0.9. §C.2 reports GRPO training hyperparameters (rollout batch 1024, group size 8, temp 1.0, KL 0.001, LR 5e-7).",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used. FARE models are standard generative evaluators without tool use, memory, or multi-step reasoning scaffolds.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "§3.1 describes data curation in detail: existing data sources with hand-crafted rubrics, synthetic data via programmatic error injection and generate-then-grade. Table 5 enumerates all data sources. §B.1 mentions N-gram decontamination.",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Raw training data, model outputs, and evaluation predictions are not released. Only aggregate benchmark numbers are reported.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "§3.1 describes data collection in detail: existing data sources are enumerated in Table 5, synthetic data generation uses 12 generators with specific sampling strategies, and rubric creation process is described with an example in §E.2.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Data comes from existing datasets and synthetic generation.",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The data pipeline is documented: Table 5 lists all sources, Fig. 2 shows breakdowns by task/domain/approach. §3.1 describes the existing data → synthetic supplement → final mix pipeline. Decontamination is mentioned in §B.1.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training data cutoff date is stated for the base models (Qwen3-8B-Base, gpt-oss-20B) or for the FARE training data collection period.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "§B.1 states 'We took efforts to decontaminate our training sets with N-gram matching approaches, following Guha et al. (2025).' This addresses potential overlap between training and evaluation data.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "While N-gram decontamination is mentioned for FARE's own training data, no analysis is provided for whether the base models (Qwen3, gpt-oss) may have seen the evaluation benchmarks during their pre-training.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference cost, latency, or tokens-per-evaluation is reported despite the paper emphasizing efficiency as a design goal (§2: 'we seek efficiency'). The SC@32 experiments multiply inference cost 32x without cost discussion.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total compute budget is stated. Training uses 2.5M samples with iterative RS-SFT but GPU hours, training time, and total API spend for synthetic data generation are not reported.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "No multi-seed results are reported. All benchmark evaluations appear to be single-run. Training uses a single random seed without sensitivity analysis.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": false,
           "justification": "The number of evaluation runs is not stated. For pairwise benchmarks with consistent accuracy, each sample is run twice (swapping order), but no repeated runs for variance estimation.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "No hyperparameter search budget is reported. The ablation study (Table 6) explores a few configurations but does not report total search budget or how the final hyperparameters were selected.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": true,
           "justification": "Table 6 shows ablation results across configurations and the paper explains selection rationale: 'we choose 40% and 60% to train FARE-8B and FARE-20B' based on peak average performance across pairwise and ProcessBench.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": true,
           "answer": false,
           "justification": "No statistical tests are performed at all, so no multiple comparison correction. The paper compares across many benchmarks and baselines without any correction.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The authors evaluate their own FARE models against baselines without acknowledging self-comparison bias. Baselines use officially reported numbers or are run with their own templates (§C.1), but the asymmetry in effort is not discussed.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": false,
           "justification": "FARE is trained on 2.5M samples with iterative RS-SFT while baselines may use far less data/compute. No compute-matched comparison is provided. FARE-20B has 3.6B active parameters vs. 70B+ baselines, but training compute is not compared.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether the 7 benchmarks actually measure meaningful evaluation quality. The paper does not question construct validity of any benchmark.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": false,
           "answer": false,
           "justification": "No scaffolding is involved in FARE evaluation. Models are prompted directly without agentic frameworks.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of temporal leakage. The base models (Qwen3, gpt-oss) may have been trained on data containing benchmark problems, and training data includes datasets from various time periods without temporal analysis.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether the evaluation setup leaks information. For example, the reference-based verification task provides a reference answer — no analysis of whether this creates an unfair advantage over baselines that don't use references.",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether training and evaluation data share structural similarities. Training data uses some of the same seed datasets (MATH, MBPP) that appear in evaluation benchmarks, and while N-gram decontamination is applied, distributional overlap is not analyzed.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": true,
           "justification": "§B.1 states 'We took efforts to decontaminate our training sets with N-gram matching approaches, following Guha et al. (2025).' This is a concrete leakage prevention method.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "FARE-20B sets a new standard for open-source evaluators, surpassing specialized 70B+ models with 3.5x fewer total parameters",
       "evidence": "Tables 1–3 show FARE-20B outperforming EvalPlanner-70B (64.29 vs 56.60 on JudgeBench), J1-70B on most pairwise benchmarks, and Qwen2.5-Math-72B-PRM on ProcessBench (84.4 vs 78.3)",
       "supported": "strong"
     },
     {
       "claim": "FARE-20B achieves near-oracle performance on MATH as an inference-time best-of-10 reranker",
       "evidence": "Figure 3 shows FARE-20B approaching the oracle performance line for MATH across multiple generators, beating SFR-Judge-70B and Skywork-Critic-70B by 14 and 21 absolute points respectively",
       "supported": "strong"
     },
     {
       "claim": "Using FARE-20B as a verifier in GRPO training improves downstream model performance by 14.1% relative to string-matching verifiers",
       "evidence": "Figure 4 shows overall accuracy improving from 34.3 (string matching) to 45.2 (FARE-20B) for Qwen2.5-7B-Base; this is a single experimental run without variance reporting",
       "supported": "moderate"
     },
     {
       "claim": "Data scaling with simple iterative RS-SFT produces evaluators competitive with RL-trained models",
       "evidence": "FARE-8B (SFT-trained) outperforms RL-trained RM-R1-14B on JudgeBench (55.71 vs 46.86); ablations in Table 6 validate individual training recipe components",
       "supported": "strong"
     },
     {
       "claim": "Continual finetuning of FARE-20B on only 15K code samples yields FARE-20B-Code that outperforms gpt-oss-120B on test-case quality evaluation",
       "evidence": "Figure 5 shows FARE-20B-Code scoring 56.55 vs gpt-oss-120B at 32.02 on test-case quality; a 76% absolute improvement from a small domain-specific fine-tuning set",
       "supported": "moderate"
     },
     {
       "claim": "Positional bias robustness emerges as a function of training data scale rather than requiring explicit algorithmic mitigation",
       "evidence": "Figure 6 shows pairwise consistency scores monotonically increasing with number of training samples for both FARE-8B and an earlier Qwen2.5-7B run",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "FARE-20B, a 20B-parameter (3.6B active) evaluator trained with iterative rejection-sampling SFT on 2.5M multi-task samples, achieves state-of-the-art performance among open-source evaluators, surpassing specialized 70B+ models on seven benchmarks. The central finding is that large-scale data curation with a simple semi-online training approach (RS-SFT) is competitive with RL-based evaluator training methods, which typically train smaller models on narrow tasks at higher computational cost. FARE shows strong downstream utility as inference-time reranker (near-oracle on MATH best-of-10), GRPO training verifier (14.1% relative improvement over string-matching), and domain-specific finetuning initialization (surpassing gpt-oss-120B on code test-case quality after only 15K samples). A secondary finding is that positional bias robustness emerges emergently with data scale rather than requiring explicit data augmentation or algorithmic interventions.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All benchmark comparisons are single-run point estimates with no variance, confidence intervals, or significance tests; small performance gaps between FARE and baselines may not be meaningful."
     },
     {
       "flag": "Training data not released",
       "detail": "The core contribution — 2.5M curated training samples — is not released, making reproduction impossible; only the public evaluation benchmarks are available."
     },
     {
       "flag": "Self-evaluation by employer",
       "detail": "All authors are Salesforce employees evaluating their own models against commercial competitors; no independent third-party reproduction or validation is provided."
     },
     {
       "flag": "Base model contamination unaddressed",
       "detail": "While FARE training data is N-gram decontaminated, the paper does not address whether Qwen3-8B-Base or gpt-oss-20B pretraining/post-training data overlaps with the seven evaluation benchmarks."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section; failure modes, generalization limits, and alternative explanations are not systematically discussed."
     },
     {
       "flag": "Vague decontamination procedure",
       "detail": "N-gram decontamination is described only as 'N-gram matching approaches' without specifying n-gram size, similarity threshold, or the number/fraction of samples removed."
     }
   ],
   "cited_papers": [
     {
       "title": "Foundational Autoraters: Taming Large Language Models for Better Automatic Evaluation",
       "relevance": "Direct predecessor establishing the foundational evaluator paradigm at scale; FARE explicitly extends and surpasses this work"
     },
     {
       "title": "Direct Judgement Preference Optimization (SFR-Judge)",
       "relevance": "Key foundational evaluator baseline used in all main benchmark tables and JETTS downstream evaluation"
     },
     {
       "title": "Self-Taught Evaluators",
       "relevance": "Closely related iterative SFT training approach; FARE directly addresses its limitations (small data scale, pairwise-only tasks)"
     },
     {
       "title": "J1: Incentivizing Thinking in LLM-as-a-Judge via Reinforcement Learning",
       "relevance": "Primary RL-trained evaluator baseline that FARE challenges with simpler SFT; directly compared on all pairwise benchmarks"
     },
     {
       "title": "CompassJudger-1: All-in-One Judge Model Helps Model Evaluation and Evolution",
       "relevance": "Key foundational evaluator baseline at multiple scales (7B, 14B, 32B) for pairwise and JETTS comparisons"
     },
     {
       "title": "ProcessBench: Identifying Process Errors in Mathematical Reasoning",
       "relevance": "Core step-level evaluation benchmark used as one of seven main evaluation benchmarks"
     },
     {
       "title": "JudgeBench: A Benchmark for Evaluating LLM-Based Judges",
       "relevance": "Primary pairwise reasoning evaluation benchmark; central to FARE's state-of-the-art claims"
     },
     {
       "title": "Evaluating Judges as Evaluators: The JETTS Benchmark",
       "relevance": "Standardized framework for inference-time reranking evaluation used in Section 4.2 downstream experiments"
     },
     {
       "title": "General-Reasoner: Advancing LLM Reasoning Across All Domains",
       "relevance": "GRPO training baseline; provides WebInstruct-Verified dataset and string-matching/General-Verifier baselines that FARE improves upon"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "FARE models are directly usable as drop-in components for RL training verifiers, inference-time rerankers, and domain-specific code evaluators — all high-value practitioner pipeline needs."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the dominant trend toward RL-based evaluator training by showing that large-scale SFT with rejection sampling matches or beats expensive RL methods."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or risk concerns raised; this is a methodology paper about training evaluation models."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild competitive positioning against RL-based approaches ('computationally demanding with brittle training pipelines') but framed constructively."
     },
     "demo_ability": {
       "score": 2,
       "justification": "If models are released (implied but not confirmed), practitioners could immediately deploy them; all evaluation prompts are provided in full in Section E."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Salesforce AI Research is a recognized lab; comparisons against GPT-5 and gpt-oss-120B as reference points add credibility and name recognition."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45657595",
         "title": "Binary Retrieval-Augmented Reward Mitigates Hallucinations",
         "points": 44,
         "comments": 3,
         "url": "https://news.ycombinator.com/item?id=45657595",
         "created_at": "2025-10-21T16:14:28Z"
       },
       {
         "hn_id": "42984225",
         "title": "Leveraging Multimodal LLM for Inspirational User Interface Search",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42984225",
         "created_at": "2025-02-08T16:52:28Z"
       },
       {
         "hn_id": "45876369",
         "title": "Diagnosing Representation Dynamics in NER Model Extension",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45876369",
         "created_at": "2025-11-10T14:30:09Z"
       }
     ],
     "top_points": 44,
     "total_points": 47,
     "total_comments": 3
   }
 }