ai-research-survey

scan.json (24276B)

---

 {
   "paper": {
     "title": "Calibrating LLM Judges: Linear Probes for Fast and Reliable Uncertainty Estimation",
     "authors": [
       "Bhaktipriya Radharapu",
       "Eshika Saxena",
       "Kenneth Li",
       "Chenxi Whitehouse",
       "Adina Williams",
       "Nicola Cancedda"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.22245"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to released code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: PPE (Frick et al., 2024), JudgeBench (Tan et al., 2024), and RewardBench (Lambert et al., 2024). These are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A.5 mentions using the 'HuggingFace transformers library' but provides no version numbers, requirements file, or detailed environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The method section and appendix describe the approach conceptually but not with sufficient operational detail to reproduce."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Standard deviations were near zero for all results' (Section 3, Evaluation metrics) but does not actually report them in the results tables. Table 7 shows standard deviations for model accuracy but not for the main calibration results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims probes outperform baselines based on comparing Kuiper/ECE values directly without any statistical significance tests (e.g., no p-values, no bootstrap tests, no paired comparisons)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 reports effect sizes in context: 'Llama-family models show the largest gains, with 70-92% improvements over multi-generation methods and 64-87% improvement over verbalized methods.' Kuiper/ECE values are reported with baselines for comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 4000 training examples and ~10K evaluation examples (Appendix A.5) but provides no justification for why these sizes are adequate, no power analysis, and no discussion of whether the sample sizes are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states results are 'averaged over three different train-test splits' and that 'Standard deviations were near zero for all results' but does not actually report variance or standard deviation values in any of the main results tables (Tables 1-3)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared: verbalized confidence, self-consistency, and majority voting (Section 3, Baselines). Additional single-pass baselines are compared in Appendix A.1 (Table 4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent methods: verbalized confidence (Xiong et al., 2024; Tian et al., 2023), self-consistency (Wang et al., 2022), and several 2024-2025 single-pass methods (CoT-kinetics, Chain of Embedding). These represent current approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablation studies are conducted: loss function ablation comparing Brier score, focal loss, and binary cross-entropy (Appendix A.7, Table 8), layer selection ablation (Appendix A.6, Figure 4), temperature ablation (A.9), number of runs ablation (A.10), and prompt ablations for verbalized confidence (A.13)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two primary calibration metrics are used: Kuiper statistic and Expected Calibration Error (ECE). AUROC is additionally used for selective classification comparisons in Appendix A.1."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed on the probe outputs. The paper relies entirely on automated metrics (Kuiper, ECE, AUROC) to assess calibration quality. Human judgment of whether the uncertainty estimates are useful in practice is not evaluated."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses explicit train-test splits from PPE datasets for in-distribution evaluation, and completely separate benchmarks (JudgeBench, RewardBench) for out-of-distribution evaluation (Section 3, Datasets)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by PPE Correctness subcategories (GPQA, IFEval, Math, MBPP, MMLU) in Appendix A.3 (Tables 5-6), and by model family, architecture type (dense vs MoE), and training paradigm (prompted vs fine-tuned)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 explicitly discusses failure cases: probes underperform on RewardBench where high baseline accuracy makes overconfident verbalized methods appear well-calibrated. The conservative estimation trade-off is discussed in detail."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that probes lag behind verbalized confidence on RewardBench (Table 2), and discusses the conservative behavior as a limitation. The abstract itself acknowledges probes 'underperform on easier datasets.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims: (1) probes achieve superior calibration (~10x savings) - supported by Tables 1-3; (2) robust OOD generalization - supported by JudgeBench results in Table 2 though RewardBench is weaker; (3) conservative estimates underperform on easier datasets - explicitly shown on RewardBench. The abstract hedges appropriately."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about why probes underperform on RewardBench (conservative estimation + high baseline accuracy) and supports this with analysis in Section 4 ('Relationship Between Accuracy and Calibration'). Ablation studies support claims about loss function and layer choices."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully bounds its claims to the tested models and datasets. It explicitly discusses where probes do and do not generalize (JudgeBench vs RewardBench), and the Limitations section acknowledges the method requires access to hidden states and labeled ground truth data."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses alternative explanations for RewardBench results (high accuracy making overconfidence appear calibrated), and considers whether the conservative behavior is inherent to the probing approach vs the training data/loss function."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies model families (LLaMA 8B/70B, Qwen 32B, J1 variants, GPT-OSS 20B, LLaMA Scout 109B) but does not provide specific model version strings or snapshot dates. 'Llama-4-Scout-17B-16E-Instruct' is mentioned in references for Scout, but for others only family names and sizes are given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in Appendix A.11 (multi-generation baselines) and A.12 (verbalized confidence) for all three judge formulations (PaV, PaS, PaL), including the complete templates with placeholder variables."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.5 reports probe hyperparameters: learning rate 10^-4, weight decay 0.01, batch size 4, 10 epochs, 4000 training examples. Baseline hyperparameters: optimal N=10, temperature=0.7 for consistency/majority (Section 3, Baselines)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method is a linear probe on model hidden states, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the datasets and their composition. Appendix A.5 specifies how training data was selected: '4000 examples randomly selected from the PPE datasets, with 2000 from all of the PPE correctness datasets and 2000 from the PPE preference dataset. The rest (~10K examples) is used for in-distribution evaluation.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section follows the Conclusion, discussing the need for hidden state access, labeled data with ground truth, and probe retraining when judge models are updated."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section provides specific threats: 'it requires access to the judge's hidden states in the middle layers and a labeled dataset with ground truth verdicts. The method doesn't work for domains where ground truth verdicts aren't available.' Also: 'if the judge is retrained or finetuned, the probe may also need to be retrained.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The limitations section discusses method requirements but does not bound the scope of the claims (e.g., it does not discuss that results are limited to pairwise judgment tasks, or that the PPE-based training data limits domain coverage)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw probe predictions, hidden state representations, and per-example results are not made available. Only aggregated metrics are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the data sources: PPE dataset from LM Arena with specific sample counts (10.2K preference, 12.7K correctness), JudgeBench (620 samples), and RewardBench (3K samples). The composition and sourcing of each dataset is described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The paper uses existing public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from data to results is documented: hidden states extracted from last token at selected layers (A.5), trained on 4000 examples, evaluated on remaining ~10K in-distribution and two OOD benchmarks. Three train-test splits are used and averaged."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. There is no mention of funding sources, grants, or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: '1FAIR at Meta, 2Meta Superintelligence Labs' (page 1)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are from Meta (FAIR and Meta Superintelligence Labs). Meta has a financial interest in demonstrating that LLM judges can be efficiently calibrated, as this supports Meta's model deployment and evaluation infrastructure. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates multiple pre-trained models (LLaMA, Qwen, GPT-OSS) on benchmarks but does not state the training data cutoff dates for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark data (PPE, JudgeBench, RewardBench) may have appeared in the training data of the judge models being evaluated."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The underlying benchmarks within PPE Correctness (MMLU-Pro, MATH, GPQA, MBPP-Plus, IFEval) are well-known public benchmarks that could plausibly appear in training data. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims '~10x computational savings' compared to multi-generation baselines and mentions 'order-of-magnitude computational savings' but does not report actual inference cost, latency, or token consumption for the probe-based approach."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report GPU hours, hardware used, total API spend, or training time for the probes."
       }
     }
   },
   "claims": [
     {
       "claim": "Linear probes trained with Brier score loss achieve superior calibration compared to verbalized confidence and multi-generation baselines across model families.",
       "evidence": "Tables 1-3 show probes achieve the lowest Kuiper/ECE scores on PPE Correctness and PPE Preference across dense and MoE models. Section 4 reports '70-92% improvements over multi-generation methods and 64-87% improvement over verbalized methods' for Llama models.",
       "supported": "strong"
     },
     {
       "claim": "Probes provide approximately 10x computational savings compared to multi-generation methods.",
       "evidence": "Section 3 states multi-generation baselines require N=10 independent samples vs. a single-pass probe. The abstract and conclusion claim '~10x computational savings.' However, no actual wall-clock time or cost measurements are provided.",
       "supported": "moderate"
     },
     {
       "claim": "Probes generalize robustly to unseen evaluation domains.",
       "evidence": "Table 2 shows probes outperform baselines on JudgeBench (OOD) across all model families. However, probes underperform on RewardBench, which the paper attributes to conservative estimation on easier datasets.",
       "supported": "moderate"
     },
     {
       "claim": "Probes produce conservative estimates that underperform on easier datasets but may benefit safety-critical deployments.",
       "evidence": "Section 4 shows probes lag on RewardBench where models have high accuracy. Appendix A.2 shows probes yield higher accuracy among the most confident predictions but assign fewer high-confidence scores. The safety-critical benefit is argued but not empirically demonstrated.",
       "supported": "moderate"
     },
     {
       "claim": "Brier score loss at the middle layer of models yields the most well-calibrated uncertainty estimates.",
       "evidence": "Appendix A.7 (Table 8) compares loss functions and shows MSE/Brier score loss outperforms focal loss variants. Appendix A.6 (Figure 4) shows middle layers outperform early/late layers.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Linear probes trained on LLM judge hidden states with Brier score loss achieve substantially better calibration than verbalized confidence and multi-generation baselines, with approximately 10x computational savings. The approach generalizes to unseen benchmarks (JudgeBench) but produces conservative estimates that underperform on easier datasets (RewardBench) where overconfident baselines appear deceptively well-calibrated. The method works across dense and Mixture-of-Experts architectures, with middle transformer layers providing the best probe performance.",
   "red_flags": [
     {
       "flag": "Company evaluating own models",
       "detail": "All authors are from Meta (FAIR and Superintelligence Labs). The J1 family of fine-tuned judges, which performs well with probes, is Meta's own product (Whitehouse et al., 2025). GPT-OSS is also an open-source Meta model. The evaluation includes Meta's models prominently."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Comparisons between probe and baseline methods are based on point estimates of Kuiper/ECE across three train-test splits. Despite claiming 'standard deviations were near zero,' no significance tests are reported and the actual standard deviations are not shown in any table."
     },
     {
       "flag": "Computational savings not quantified",
       "detail": "The paper repeatedly claims '~10x computational savings' but this is derived from the theoretical N=10 generation count of consistency baselines, not from actual wall-clock time or cost measurements. The overhead of extracting and storing hidden states is not quantified."
     },
     {
       "flag": "No code release",
       "detail": "Despite presenting a method claimed to be a practical 'plug-and-play solution for deployment at scale,' no code is released, making it difficult for practitioners to verify or adopt the approach."
     }
   ],
   "cited_papers": [
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "Foundational work on the LLM-as-judge paradigm used for evaluating model outputs, directly relevant to the survey's coverage of LLM evaluation methodology."
     },
     {
       "title": "Can LLMs Express Their Uncertainty? An Empirical Evaluation of Confidence Elicitation in LLMs",
       "authors": ["Miao Xiong", "Zhiyuan Hu", "Xinyang Lu"],
       "year": 2024,
       "relevance": "Empirical evaluation of LLM confidence calibration methods, relevant to assessing how well models can self-assess their outputs."
     },
     {
       "title": "Trust or Escalate: LLM Judges with Provable Guarantees for Human Agreement",
       "authors": ["Jaehun Jung", "Faeze Brahman", "Yejin Choi"],
       "year": 2024,
       "relevance": "Addresses the reliability of LLM judges with uncertainty-aware routing, directly relevant to the survey's evaluation of LLM judge methodology."
     },
     {
       "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational work on self-consistency as an uncertainty estimation method for LLMs, relevant to understanding multi-generation approaches to calibration."
     },
     {
       "title": "J1: Incentivizing Thinking in LLM-as-a-Judge via Reinforcement Learning",
       "authors": ["Chenxi Whitehouse", "Tianlu Wang", "Ping Yu"],
       "year": 2025,
       "relevance": "State-of-the-art fine-tuned LLM judge family evaluated in this paper, relevant to the survey's coverage of LLM judge quality."
     },
     {
       "title": "RewardBench: Evaluating Reward Models for Language Modeling",
       "authors": ["Nathan Lambert", "Valentina Pyatkin", "Jacob Morrison"],
       "year": 2024,
       "arxiv_id": "2403.13787",
       "relevance": "Benchmark for evaluating reward models, relevant to the survey's coverage of LLM evaluation benchmarks and methodology."
     },
     {
       "title": "JudgeBench: A Benchmark for Evaluating LLM-Based Judges",
       "authors": ["Sijun Tan", "Siyuan Zhuang", "Kyle Montgomery"],
       "year": 2024,
       "relevance": "Benchmark specifically designed for evaluating LLM judge quality, relevant to understanding evaluation methodology for LLM-as-judge systems."
     },
     {
       "title": "Semantic Entropy Probes: Robust and Cheap Hallucination Detection in LLMs",
       "authors": ["Jannik Kossen", "Jiatong Han", "Muhammed Razzak"],
       "year": 2024,
       "arxiv_id": "2406.15927",
       "relevance": "Related interpretability-based approach for uncertainty estimation in LLMs, relevant to the survey's coverage of LLM reliability methods."
     },
     {
       "title": "Overconfidence in LLM-as-a-Judge: Diagnosis and Confidence-Driven Solution",
       "authors": ["Zailong Tian", "Zhuoheng Han", "Yanzhe Chen"],
       "year": 2025,
       "relevance": "Directly addresses LLM judge overconfidence, a key reliability concern in the LLM evaluation pipeline."
     },
     {
       "title": "Discovering Latent Knowledge in Language Models Without Supervision",
       "authors": ["Collin Burns", "Haotian Ye", "Dan Klein", "Jacob Steinhardt"],
       "year": 2023,
       "relevance": "Foundational work on extracting knowledge from LLM internal representations, relevant to interpretability-based approaches for LLM reliability."
     },
     {
       "title": "How to Evaluate Reward Models for RLHF",
       "authors": ["Evan Frick", "Tianle Li", "Connor Chen"],
       "year": 2024,
       "relevance": "Source of the PPE dataset used for training and evaluation in this work, relevant to understanding evaluation methodology for RLHF reward models."
     }
   ]
 }