ai-research-survey

calibration.json (20137B)

---

 {
   "paper_slug": "advancing-language-model-2025",
   "calibration_date": "2026-02-28",
   "calibration_model": "opus",
   "total_questions": 50,
   "agreement_count": 48,
   "disagreement_count": 2,
   "agreement_rate": 0.96,
   "disagreements": [
     {
       "category": "evaluation_design",
       "question": "multiple_metrics",
       "sonnet_applies": true,
       "sonnet_answer": true,
       "opus_applies": true,
       "opus_answer": false,
       "direction": "sonnet_generous",
       "explanation": "The paper explicitly states 'Accuracy (Pass@1) is used as the primary evaluation metric' and reports only this single metric across all four benchmarks (MATH500, AIME, Omni-MATH-500, GPQA). The schema requires 'at least two different metrics (e.g., accuracy AND F1, or Pass@1 AND Pass@10).' Using the same metric (accuracy) on multiple datasets is not the same as using multiple metrics. Sonnet appears to have conflated multiple benchmarks with multiple metrics. The paper also tracks training-time metrics (response length, KL divergence, reward) but these are process metrics, not evaluation metrics of the system's output quality."
     },
     {
       "category": "setup_transparency",
       "question": "model_versions_specified",
       "sonnet_applies": true,
       "sonnet_answer": true,
       "opus_applies": true,
       "opus_answer": false,
       "direction": "sonnet_generous",
       "explanation": "While T1's own base models are specified (GLM-4-9B, Qwen2.5-14B, Qwen2.5-32B — identifiable checkpoints), the comparison baselines in Table 1 use marketing names without version specification: 'GPT-4o' and 'Claude-3.5-sonnet' lack snapshot dates or API versions. The schema explicitly states: 'Marketing names like \"Gemini-2.5\" or \"GPT-4o\" without a snapshot date or API version do NOT count as specified versions.' Since these baseline model results appear in the main comparison table and their exact versions affect reproducibility, this criterion is not satisfied."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The model weights and the data for SFT and RL training are publicly available at https://github.com/THUDM/T1.' A working GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states SFT and RL training data are publicly available at the GitHub repository. Training data sourced from publicly available datasets (MATH-train, NuminaMath). Evaluation benchmarks (AIME2024, MATH, GPQA, Omni-MATH) are also public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions SGLANG as the inference engine and provides training hyperparameters, but does not provide a requirements.txt, Dockerfile, conda environment, or detailed list of library versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A.1 provides training hyperparameters (learning rates, K, temperature, etc.) but does not include step-by-step reproduction instructions, README-level commands, or scripts to replicate the main experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 1 are reported as single accuracy values (e.g., '92.4') with no confidence intervals, error bars, or uncertainty quantification. AIME is evaluated 32 times and averaged, but no variance or CI is reported for these repeated evaluations."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims T1 outperforms QwQ-32B-Preview and other baselines based solely on comparing point estimates. No statistical significance tests (p-values, t-tests, bootstrap tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute percentage improvements with baseline context: 'over a 10% improvement on Omni-MATH-500' and 'over a 20% improvement on AIME over its T1-SFT version,' with both baseline and T1 scores provided in Table 1 for comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "AIME uses only 30 problems. Omni-MATH-500 uses 500 examples described as 'efficient yet comprehensive evaluation' without statistical justification. No power analysis or formal sample size justification is provided for any benchmark."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "AIME is evaluated 32 times and averaged 'to get stable results,' but the standard deviation or other spread measure across these 32 runs is not reported. All other benchmarks appear to be single-run results. No variance, std dev, or IQR is reported anywhere."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 includes multiple baselines: GPT-4o, Claude-3.5-sonnet, Llama-3.3-70B-Instruct, Qwen2.5-Math-7B-Instruct, o1-preview, QwQ-32B-preview, and the original base models plus T1-SFT ablations."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary systems from late 2024/early 2025: QwQ-32B-Preview, o1-preview, Claude-3.5-Sonnet, and GPT-4o, representing the state of the art at the time."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes systematic ablation studies: number of sampled responses K (Figure 3, Figure 4), sampling temperature (Table 2), penalty mechanism (Table 3), and SFT vs. SFT+RL (Table 1). Individual components are isolated and their contribution measured."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly states 'Accuracy (Pass@1) is used as the primary evaluation metric' and this is the only evaluation metric reported. The same metric (accuracy) is applied across four benchmarks (MATH500, AIME, Omni-MATH-500, GPQA). Using the same metric on multiple datasets is not the same as using multiple different metrics (e.g., Pass@1 AND Pass@10, or accuracy AND F1). Training metrics like response length and KL divergence are process metrics, not evaluation metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper for mathematical reasoning where correctness is determined by matching against ground-truth answers. Human evaluation of model outputs is not applicable."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training data uses MATH-train and NuminaMath; evaluation uses MATH-test/MATH500 (a separate split), AIME2024 (competition problems), Omni-MATH-500, and GPQA. Appendix A.1 confirms the train/test split: '12k for the SFT stage and the others for RL training.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across four benchmarks of different difficulty levels (competition math AIME, olympiad math Omni-MATH, school math MATH500, and science GPQA). Figure 7 shows inference scaling separately for each dataset."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents a successful case study (Figure 9, AIME 2024 Question 15) but does not discuss failure cases, error patterns, or provide qualitative analysis of where T1 fails. No error analysis or examples of incorrect reasoning are shown."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: min-p sampling causes training collapse (Table 2, MATH500 drops from 86.4 to 78.8), low temperatures (<=1.0) often cause training collapse, training without penalty leads to degradation and eventually fails (Table 3), and excessively high temperature (1.3) hurts performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims T1 'achieves superior performance on challenging math reasoning benchmarks' — supported by Table 1 showing SOTA on MATH500 and Omni-MATH-500. The abstract claims T1 'exhibits inference scaling behavior' — supported by Figures 6-7 showing monotonic improvement with more thinking tokens."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims (e.g., 'sampling more responses encourages exploration,' 'penalty prevents collapse') are supported by controlled single-variable ablation studies: varying only K (Figure 3), only temperature (Table 2), only penalty (Table 3). This controlled manipulation design is adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Advancing Language Model Reasoning' without bounding to math. The abstract claims the approach enables 'effective test-time scaling' and the conclusion says it enhances 'reasoning capabilities' broadly. But evaluation is limited to math benchmarks (MATH, AIME, Omni-MATH) and GPQA science. No code generation, commonsense reasoning, or other reasoning domains are tested. The paper notes OOD generalization to GPQA but does not explicitly bound its broader claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements. It does not consider whether improvements could be due to data overlap, whether the reward signal design alone (vs. exploration strategies) accounts for the gains, or whether the summarization model used in inference scaling analysis introduces confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "T1's base models are specified as GLM-4-9B, Qwen2.5-14B, and Qwen2.5-32B (identifiable checkpoints). However, comparison baselines in Table 1 use marketing names: 'GPT-4o' and 'Claude-3.5-sonnet' without snapshot dates or API versions. The schema explicitly states 'Marketing names like \"Gemini-2.5\" or \"GPT-4o\" without a snapshot date or API version do NOT count as specified versions.' Some baseline results are reported 'from previous works if available,' but the versions remain unspecified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 2.2.1 describes the prompting procedure in natural language: 'we prompt an LLM to thoroughly examine each attempt' and 'we further prompt an LLM to incorporate these refined attempts into a single output.' The actual prompt text used is not provided. The schema requires actual prompt text, not natural language descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 reports: SFT learning rate 1e-5 with cosine decay, RL learning rate 1.5e-6, KL coefficient 2e-4, K=64 responses, temperature=1.2, top-p=0.95, max generation length 10240/16384, reward function (binary correctness), gradient updates every 32 prompts."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "T1 is a training methodology (SFT + RL), not an agentic scaffolding system. The model generates reasoning chains end-to-end in a single forward pass. No external tool use, memory, or multi-step scaffolding is involved."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 documents data preprocessing: answer extraction from NuminaMath using an LLM, data filtering to retain instances with pass rate in (0, 0.3) from 16 generated responses per question, resulting in 30k RL training examples and 12k SFT examples. Filtering criteria and resulting sizes are stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has an 'Impact Statement' section but no dedicated Limitations or Threats to Validity section. The Impact Statement is entirely generic boilerplate: 'The ethical aspects and societal implications of our work align with those commonly associated with advancing the field of machine learning.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. The Impact Statement contains no methodological discussion. The Conclusion also lacks any discussion of limitations or threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. While it mentions T1 is 'primarily optimized for math-related tasks,' it does not explicitly enumerate scope boundaries or disclaim broader generalization of the claimed 'reasoning capabilities.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Model weights and training data are released at https://github.com/THUDM/T1. Evaluation benchmarks (AIME2024, MATH, GPQA, Omni-MATH) are publicly available, enabling independent verification and re-evaluation."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 describes data collection: training data from MATH-train (Hendrycks et al., 2021) and NuminaMath (Li et al., 2024b), with answer extraction and filtering procedures. Evaluation datasets and their sources are specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. All training and evaluation data come from publicly available mathematical datasets. This criterion is not applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 documents the data pipeline: raw data (NuminaMath + MATH-train) → answer extraction via LLM → data filtering (pass rate in (0, 0.3) from 16 generated responses) → 12k SFT split + 30k RL split. Transformation steps and resulting sizes are stated at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section discloses funding from NSFC 62495063, Tsinghua University-Siemens Joint Research Center (JCIIOT), and the New Cornerstone Science Foundation (XPLORER PRIZE)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed in the paper header: Tsinghua University (Hou, Lu, Zhang, Li, Yao, Li, Tang, Dong) and ZhipuAI (Lv). The ZhipuAI affiliation of co-author Xin Lv is disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders (NSFC, Tsinghua-Siemens joint research center, New Cornerstone Science Foundation) are academic/government funding bodies without a direct financial stake in T1's performance on math benchmarks. Although co-author Xin Lv is from ZhipuAI (maker of GLM models), ZhipuAI is not listed as a funder."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. Co-author Xin Lv is affiliated with ZhipuAI, whose GLM-4-9B model is one of three base models evaluated, but this potential conflict of interest is not explicitly declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for the base models (Qwen2.5, GLM-4) or the comparison models (GPT-4o, Claude-3.5-Sonnet, o1-preview, QwQ-32B-Preview). This information is needed to assess contamination risk."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. The paper does not analyze whether MATH test examples, AIME2024 problems, or Omni-MATH problems could have appeared in the pre-training data of the base models (Qwen2.5, GLM-4)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MATH benchmark (Hendrycks et al., 2021) was published in 2021. The base models (Qwen2.5, GLM-4) were likely trained on data that includes MATH problems. AIME2024 is newer but competition problems circulate online. No contamination analysis or discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Pre-registration is not applicable."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Demographics are not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Inclusion/exclusion criteria are not applicable."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Randomization of participants is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports token counts for inference scaling analysis (up to ~6000 thinking tokens per response) and max generation lengths (10240-16384 tokens), but does not report wall-clock time, API costs, or cost per example for running T1."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report GPU hours, hardware used (GPU type/count), total training time, or total compute budget. Training a 32B model with K=64 sampled responses per prompt is expensive, but the compute requirements are not quantified."
       }
     }
   }
 }