ai-research-survey

scan.json (27009B)

---

 {
   "paper": {
     "title": "On Calibration of Large Language Models: From Response To Capability",
     "authors": ["Sin-Han Yang", "Cheng-Kuang Wu", "Chieh-Yen Lin", "Yun-Nung Chen", "Hung-yi Lee", "Shao-Hua Sun"],
     "year": 2026,
     "venue": "Preprint",
     "arxiv_id": "2602.13540"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Source code: https://github.com/appier-research/llm-calibration' providing a GitHub URL for the code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: TriviaQA, SimpleQA, GSM8K, MATH-500, AIME25, MMLU, and GPQA. All are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies model names and sampling hyperparameters (Appendix B.2) and probe training hyperparameters (Table 5), but does not provide a requirements.txt, Dockerfile, or detailed environment setup section listing library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the paper provides a GitHub link and detailed descriptions of methods and hyperparameters, there are no explicit step-by-step reproduction instructions (e.g., README with commands) within the paper itself. The code repository may contain them, but the paper does not."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 shows 95% confidence intervals for the expected accuracy estimation under different keval values. The pass@k simulation in Figure 9 also shows confidence interval bands. However, the main results in Table 2 report only point estimates without uncertainty."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'probing consistently outperforms the random baseline', 'Probe-MATH outperforms Oracle-RC') but relies solely on comparing Brier score values without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw Brier scores and MSE values. No standardized effect sizes (Cohen's d, relative improvement with baseline context) are reported. The reader can compute relative differences from the tables, but the paper does not contextualize the magnitude of improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3.1 and Appendix B.1 provide a thorough justification for the choice of keval=100, including theoretical analysis of binomial variance, margin of error calculations, and empirical sensitivity analysis (Figure 6)."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Table 2 report single Brier score values without variance, standard deviation, or confidence intervals across runs. There is no mention of running experiments with multiple random seeds or reporting spread measures for the main calibration results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes a uniform random baseline with analytical expected loss (Section 4.2), and compares against verbalized confidence, P(True), and response consistency methods (Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent methods: verbalized confidence (Tian et al., 2023), P(True) (Kadavath et al., 2022), response consistency (Wang et al., 2022), and probing (Li et al., 2021). The paper adapts these to the capability calibration setting and uses recent models (Olmo-3, Qwen3, gpt-oss-20b)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper conducts multiple ablation-like studies: effect of keval (Appendix B.1), effect of kc for response consistency (Appendix C.3, Table 6), effect of training dataset domain on probe generalization (Table 2, Table 7), and mixed vs. single-dataset probes (Appendix C.4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Brier score as the primary calibration metric, but also evaluates downstream application performance via MSE for pass@k simulation (Table 3) and success rate for inference budget allocation (Figure 5). The paper explains why it chose Brier score over ECE (Section 4.1)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a calibration framework paper evaluating confidence estimation methods on standard benchmarks. Human evaluation of system outputs is not relevant to the claims about calibration quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For probing experiments, the paper uses separate training, validation, and test splits. For MATH, 'we use MATH-500 as the test set, and the remaining 12,000 instances as the training and validation sets' (Appendix C.2). Out-of-distribution generalization is tested on entirely separate datasets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-dataset breakdowns across 7 datasets for all methods and 3 models. Results are broken down by domain (factual knowledge, mathematical reasoning, general exams) and by in-domain vs. out-of-domain generalization."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses when probing fails to generalize to out-of-domain settings (Section 4.3.2), when verbalized confidence fails (worse than random baseline for some models, Table 2), and when Probe-MATH does not fit well for pass@k simulation (Figure 9, Appendix D.1.2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 'Olmo-3-7B-Instruct and Qwen3-8B do not even consistently outperform the random uniform baseline with verbalized confidence' (Section 4.3.2). Probing does not consistently generalize to out-of-domain settings. 2-layer MLP probes were tried but 'did not differ from linear probes' (Appendix C.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that capability calibration differs from response calibration theoretically and empirically (supported by Theorems 1-2 and Figure 2), that capability-calibrated confidence improves pass@k prediction (supported by Table 3) and inference budget allocation (supported by Figure 5). All claims are supported by results in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation-style comparisons (e.g., changing the training dataset for probes, varying keval, varying kc). These are controlled single-variable manipulations. The theoretical claims (Theorems 1-2) are proved mathematically. The paper's main claims are about the distinction between definitions rather than causal improvement claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly tests and discusses generalization boundaries: in-domain in-distribution, in-domain out-of-distribution, and out-of-domain settings (Table 2, Section 4.3.2). It acknowledges that 'Developing generalizable methods for capability calibration remains an important direction for future work.' The title is appropriately scoped."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for its results. For example, it does not consider whether the advantage of probing is due to memorizing dataset-specific patterns rather than capturing true capability, or whether the theoretical framework's assumptions (e.g., deterministic correctness function) limit its applicability. The Impact Statement section does not address these concerns."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'Olmo-3-7B-Instruct', 'Qwen3-8B', and 'gpt-oss-20b'. These are model family names with size designations but not specific version snapshots or API version identifiers. No snapshot dates or checkpoint hashes are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided for verbalized confidence (Figure 7) and P(True) (Figure 8) in the appendix. For probing, prompts are not needed as it operates on model activations. The prompts include exact wording with placeholders (e.g., '{question}') whose fill values are the benchmark questions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Sampling hyperparameters are reported in Appendix B.2 (temperature, top-p for each model). Probe training hyperparameters are comprehensively reported in Table 5 (epochs, batch size, weight decay, learning rate, loss function, pooling method, feature standardization)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. It evaluates confidence estimation methods applied to LLM outputs, not agent-based systems."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents data splits (Appendix C.2): TriviaQA and GSM8K use their training sets; MATH uses MATH-500 as test and remaining 12,000 as training/validation. Dataset selection criteria are described in Section 4.3.1 with clear rationale for each domain and difficulty level."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not have a dedicated limitations or threats-to-validity section. Future directions are mentioned briefly in Section 6 (Conclusion), but there is no substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper acknowledges that probing does not generalize to out-of-domain settings (Section 4.3.2) and that Probe-MATH does not fit well for pass@k simulation (Appendix D.1.2), but these are presented as results rather than as threats to the validity of the overall framework."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Section 5.3 mentions that 'a comprehensive empirical evaluation of all downstream tasks is beyond the scope of this work,' but this is the only scope boundary discussed. No discussion of what populations, task types, or model families the results cannot generalize to."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used are publicly available benchmarks (TriviaQA, SimpleQA, GSM8K, MATH-500, AIME25, MMLU, GPQA). Source code is released at the provided GitHub link, which could contain generated model outputs for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3.1 describes the data collection procedure: datasets selected from three task domains (factual knowledge, mathematical reasoning, general exams) with explicit justification for each choice. For capability calibration targets, keval=100 samples are drawn per query with detailed justification (Appendix B.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: sampling keval=100 responses per query, computing correctness via deterministic evaluator C, estimating expected accuracy, training probes on hidden states, and evaluating with Brier score. Appendix B.1, C.2, and the main paper describe each step."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section discloses funding: 'This work was supported in part by the National Science and Technology Council, Taiwan, under the Grant 114-2628-E-002-021-, and the Taiwan Centers of Excellence. Shao-Hua Sun was supported by the Yushan Fellow Program of the Ministry of Education, Taiwan.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Appier AI Research and National Taiwan University. The affiliations are listed in the header of the paper."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders are the National Science and Technology Council of Taiwan and the Ministry of Education of Taiwan. These are government agencies with no direct financial interest in the outcome of calibration research. Authors from Appier could have a commercial interest, but the paper evaluates open models, not Appier products."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. Some authors are affiliated with Appier (a commercial AI company), but no declaration of competing interests is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates LLMs on benchmarks (TriviaQA, GSM8K, MATH-500, MMLU, GPQA, etc.) but does not state the training data cutoff dates for any of the three models used (Olmo-3-7B-Instruct, Qwen3-8B, gpt-oss-20b)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the benchmark datasets (many of which are publicly available and widely used) may have been included in the training data of the evaluated models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Several benchmarks used (TriviaQA from 2017, GSM8K from 2021, MMLU from 2020, MATH from 2021) were published well before the models' training. No contamination analysis is provided despite this risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference cost is a central dimension of the analysis. Figure 3 and 4 plot cost-performance tradeoffs. Table 2 lists relative cost per method. The paper discusses that probing costs 'less than decoding a single token' and response consistency 'costs more than decoding the response itself' (Sections 4.2, 4.3.2)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The total computational budget is not stated. The paper used keval=100 samples across 7 datasets for 3 models, which represents substantial compute, but no total GPU hours, API costs, or hardware specifications are provided. Appendix B.2 mentions 'limited computational budget' for Qwen3-8B but does not quantify it."
       }
     }
   },
   "claims": [
     {
       "claim": "Capability calibration is theoretically and empirically distinct from response calibration.",
       "evidence": "Theorem 1 proves distinct optimal confidence estimators. Theorem 2 shows the loss decomposition with output correctness variance term. Figure 2 and Appendix E empirically confirm that response-level correctness does not reflect expected accuracy across models and datasets.",
       "supported": "strong"
     },
     {
       "claim": "Linear probing on LLM hidden states provides the best cost-performance tradeoff for capability calibration.",
       "evidence": "Table 2 and Figures 3-4 show that probing consistently outperforms the random baseline at less-than-one-token inference cost. Probing achieves in-domain in-distribution Brier scores competitive with or better than other methods across all 3 models and 7 datasets.",
       "supported": "moderate"
     },
     {
       "claim": "Capability-calibrated confidence outperforms response-calibrated confidence for pass@k simulation.",
       "evidence": "Table 3 shows that Oracle-CC achieves near-zero MSE for pass@k simulation, while Oracle-RC's MSE increases with k. Probe-MATH outperforms Oracle-RC across all models and k values on MATH-500.",
       "supported": "strong"
     },
     {
       "claim": "Capability-calibrated confidence improves inference budget allocation over uniform allocation.",
       "evidence": "Figure 5 and Appendix D.2.2 (Figure 10) show that greedy allocation using capability-calibrated confidence (Oracle, Probe-MATH, Verbalized) outperforms uniform allocation across compute budgets on MATH-500 and AIME25.",
       "supported": "strong"
     },
     {
       "claim": "Probing generalizes reasonably well under in-domain out-of-distribution settings but not consistently to out-of-domain settings.",
       "evidence": "Table 2 shows that probes trained on TriviaQA generalize well to SimpleQA (same factual knowledge domain) across models, but performance on out-of-domain tasks (e.g., math probes on factual knowledge) is inconsistent.",
       "supported": "moderate"
     },
     {
       "claim": "Verbalized confidence effectiveness is model-dependent.",
       "evidence": "Table 2 shows gpt-oss-20b achieves best or second-best performance with verbalized confidence, while Olmo-3-7B-Instruct and Qwen3-8B do not consistently outperform the random baseline with the same method.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "The paper introduces capability calibration, which targets a model's expected accuracy on a query (rather than single-response correctness), and proves it is theoretically distinct from response calibration via a loss decomposition involving output correctness variance. Experiments across 3 LLMs and 7 datasets show that linear probing on model hidden states offers the best cost-performance tradeoff for capability calibration, achieving meaningful calibration at less than single-token inference cost. Capability-calibrated confidence is shown to improve pass@k prediction (outperforming oracle response calibration) and inference budget allocation (outperforming uniform allocation).",
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "The paper evaluates models on benchmarks like TriviaQA (2017), GSM8K (2021), MMLU (2020), and MATH (2021) without discussing whether these datasets may be in the models' training data. Contamination could affect the expected accuracy estimates and invalidate the calibration measurements."
     },
     {
       "flag": "No significance testing",
       "detail": "All comparative claims are based on comparing raw Brier scores without statistical tests. Given the stochastic nature of the evaluation (sampling keval=100 responses per query), the differences between methods could be within noise."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section, which is unusual for a venue-targeted ML paper. Key limitations (e.g., deterministic correctness function assumption, limited model diversity with only 3 open-weight models, limited task coverage) are not discussed."
     },
     {
       "flag": "Model version ambiguity",
       "detail": "Model versions are specified only by name and size (e.g., 'Olmo-3-7B-Instruct', 'gpt-oss-20b') without specific checkpoint identifiers or snapshot dates, making exact reproduction difficult."
     }
   ],
   "cited_papers": [
     {
       "title": "A survey of confidence estimation and calibration in large language models",
       "authors": ["J. Geng", "F. Cai", "Y. Wang", "H. Koeppl", "P. Nakov", "I. Gurevych"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLM confidence estimation and calibration methods, directly relevant to understanding the state of the art this paper builds upon."
     },
     {
       "title": "Can llms express their uncertainty? an empirical evaluation of confidence elicitation in llms",
       "authors": ["M. Xiong", "Z. Hu", "X. Lu", "Y. Li", "J. Fu", "J. He", "B. Hooi"],
       "year": 2023,
       "arxiv_id": "2306.13063",
       "relevance": "Empirical study of confidence elicitation methods in LLMs, a key baseline approach that this paper extends to capability-level calibration."
     },
     {
       "title": "Language models (mostly) know what they know",
       "authors": ["S. Kadavath", "T. Conerly", "A. Askell"],
       "year": 2022,
       "arxiv_id": "2207.05221",
       "relevance": "Introduces the P(True) confidence estimation method evaluated as a baseline in this paper."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["X. Wang", "J. Wei", "D. Schuurmans", "Q. Le", "E. Chi", "S. Narang", "A. Chowdhery", "D. Zhou"],
       "year": 2022,
       "arxiv_id": "2203.11171",
       "relevance": "Introduces self-consistency/response consistency as a confidence estimation strategy, evaluated as a baseline method."
     },
     {
       "title": "Learning how hard to think: Input-adaptive allocation of lm computation",
       "authors": ["M. Damani", "I. Shenfeld", "A. Peng", "A. Bobu", "J. Andreas"],
       "year": 2024,
       "arxiv_id": "2410.04707",
       "relevance": "Proposes inference budget allocation framework that directly leverages expected accuracy, used as a downstream application for capability calibration."
     },
     {
       "title": "Scaling llm test-time compute optimally can be more effective than scaling model parameters",
       "authors": ["C. Snell", "J. Lee", "K. Xu", "A. Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Studies optimal test-time compute scaling for LLMs, directly relevant to the inference budget allocation application."
     },
     {
       "title": "How do large language monkeys get their power (laws)?",
       "authors": ["R. Schaeffer", "J. Kazdan", "J. Hughes"],
       "year": 2025,
       "arxiv_id": "2502.17578",
       "relevance": "Studies pass@k scaling in LLMs and AI safety implications, directly motivating the pass@k simulation application."
     },
     {
       "title": "Efficient prediction of pass@k scaling in large language models",
       "authors": ["J. Kazdan", "R. Schaeffer", "Y. Allouah"],
       "year": 2025,
       "arxiv_id": "2510.05197",
       "relevance": "Proposes methods for predicting pass@k performance, a direct comparison point for the pass@k simulation application."
     },
     {
       "title": "On calibration of modern neural networks",
       "authors": ["C. Guo", "G. Pleiss", "Y. Sun", "K. Q. Weinberger"],
       "year": 2017,
       "relevance": "Foundational work on calibration of neural networks that establishes the response calibration framework this paper extends."
     },
     {
       "title": "Beyond the singular: The essential role of multiple generations in effective benchmark evaluation and analysis",
       "authors": ["W. Zhang", "H. Cai", "W. Chen"],
       "year": 2025,
       "arxiv_id": "2502.08943",
       "relevance": "Argues for using multiple generations in LLM evaluation, directly supporting the motivation for capability calibration."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces pass@k evaluation methodology and unbiased estimator used in this paper's pass@k simulation experiments."
     },
     {
       "title": "RouteLLM: Learning to route llms with preference data",
       "authors": ["I. Ong", "A. Almahairi", "V. Wu"],
       "year": 2024,
       "arxiv_id": "2406.18665",
       "relevance": "Proposes LLM routing methods that could benefit from capability calibration as discussed in the paper's applications section."
     }
   ]
 }