ai-research-survey

scan.json (24715B)

---

 {
   "paper": {
     "title": "Predicting LLM Reasoning Performance with Small Proxy Model",
     "authors": ["Woosung Koh", "Juyoung Suk", "Sungjun Han", "Se-Young Yun", "Jamin Shin"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.21013",
     "doi": "10.48550/arXiv.2509.21013"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RBRIDGE enables small proxy models (≤1B) to predict large-model (13B-32B) reasoning performance by aligning evaluation with the pre-training objective (NLL) and the target task (using frontier model reasoning traces as gold labels with automatic token weighting). It achieves 100x+ compute savings for dataset ranking, strongest proxy-target correlation across six benchmarks, and successful zero-shot functional relationship transfer across pre-training datasets.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL is provided. The paper mentions open-sourcing the dataset but no link to code is given."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we plan to open-sourced our dataset' (§5) and uses publicly available benchmarks (GSM8K, MATH500, ARC-C, etc.) and OLMo checkpoints. The frontier model reasoning traces are described as planned for release."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.1 specifies hardware: 'A100 80G, H100 and H200 nodes' and '256 H100 GPUs with HBM3' for pre-training. However, no software dependencies or library versions are listed, but hardware is detailed."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides pseudocode (Algorithm 1 in Appendix B), the exact prompt used, and states the method is 'fully reproducible using the information provided in this paper' (§7 Reproducibility Statement). Experimental protocols reference open-source assets from Magnusson et al. (2025)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Error bars are shown in Fig. 4 ('Error bars indicate one standard deviation'), Fig. 7a uses box-and-whisker plots showing distributional spread, and Fig. 12 shows error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Comparisons between RBRIDGE and baselines are made by comparing R² and MAE values directly without any significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported contextually: e.g., '27% higher DAcc.' (§4.2(i)), '100.2× to 733.4× less FLOPs' (§4.2(i)), '74.7% NLL decline' (§3.2). R² and MAE values provide magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for sample sizes. The number of pre-training checkpoints (15 data points at 250B intervals), the choice of 25 datasets, and the number of benchmarks (6) are not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "5-fold cross-validation is used for experiments (ii) and (iii), and experiment (i) uses proxy models averaged across three pre-training seeds (§4.1(i)). Standard deviation shown in Fig. 4 error bars."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five baselines for dataset ranking (§4.1(i)) and six baselines for proxy-target relationship (§4.1(ii)) are compared, including Acc./p@1, iSFT, TED, MPCA, NLL, and Rϕ."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary methods: ScalingBench (Xiao et al., 2024), DataDecide (Magnusson et al., 2025), iSFT (Snell et al., 2024), and TED (Schaeffer et al., 2023)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Fig. 7b shows an ablation study decomposing RBRIDGE into its components: Rϕ → +RBRIDGE NLL → +Normalization, showing each contributes consistent improvement across all three experimental settings."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: R², MAE, Decision Accuracy, Kendall's Tau (Appendix D), and evaluation spans six benchmarks covering math, science, engineering, commonsense, and coding."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant — this is a computational methodology for predicting model performance via proxy metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "5-fold cross-validation is used (§4.1(ii)), reporting both train R² and test MAE. Experiment (iii) uses a separate pre-training dataset D' as held-out evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (6 benchmarks in Tab. 2, Tab. 5, Tab. 6) rather than just aggregate averages."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure cases: one outlier in zero-shot transfer (CQA MAE=9.716 in Tab. 3), and limitations where frontier models fail to produce outputs in required format (§5.1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Negative results are reported: Fig. 2 shows small models giving wrong direction slopes, Fig. 3b shows OOD gold labels providing no signal, and Tab. 1 shows ScB performing worse than Rϕ."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims — 100x compute reduction, strongest correlation across six benchmarks, zero-shot transfer — are all supported by results in Tab. 2, Fig. 6, and Tab. 3 respectively."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'alignment improves performance' are supported by ablation study (Fig. 7b) showing controlled removal of components. The paper uses controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds claims to specific model sizes (1B→13B, 1B→32B), specific datasets (OLMo-Mix-1124), and acknowledges limitations on scale ('larger-scale studies across more model sizes and pre-training datasets would be ideal', §4.1(iii))."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses distributional alignment as an alternative explanation (§3.1), analyzes why existing approaches fail along two axes, and considers that frontier model imperfections could affect results (§5.1)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly frames RBRIDGE as a proxy metric for target-scale accuracy and discusses the gap between proxy and target metrics throughout (§2 Problem Setting defines the proxy-target relationship formally)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'we use GPT 4o to generate Rϕ' (Appendix C.3) without specifying a version or snapshot date. Model sizes (1B, 7B, 13B, 32B) are given but these are OLMo models referenced by size, not version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The exact prompt used to generate reasoning traces is provided in Appendix B: 'System: You are a helpful assistant that solves [task] problems. User: [question] Respond ONLY with a JSON object...'"
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Greedy decoding is specified for Rϕ generation (Appendix B). SFT hyperparameters are in Tab. 4 (learning rate, warmup ratio, batch size, epochs). Pre-training follows OLMo 2 settings."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. RBRIDGE is a metric computation, not an agent system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The alternative dataset D' composition is described (Appendix C.4): '8.5:1:0.5 ratio of English:multilingual:math/code' with specific dataset sources. Frontier model output extraction is documented (discard answer, keep reasoning trace)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 'Limitation and Future Direction' discusses three specific limitations of RBRIDGE."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "§5.1 discusses specific threats: frontier models don't achieve perfect accuracy on reasoning tasks, format failures in reasoning trace extraction, and the framework for practical application remains an open challenge."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges scope limits: 'larger-scale studies across more model sizes and pre-training datasets would be ideal' (§4.1(iii)), not tested on long CoT models (Appendix C.3), and HumanEval excluded from experiment (iii) due to 0% p@1."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental data (individual checkpoint results, per-example scores) are not released. Only aggregated results in tables and figures are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is well-described: benchmarks are standard public datasets, pre-training uses OLMo-Mix-1124 with checkpoints at 250B intervals, and the alternative dataset composition is specified in Appendix C.4."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks and pre-training datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: frontier model generates reasoning traces → extract Rϕ → compute letter-level probabilities → aggregate to token weights → compute weighted NLL (Algorithm 1, Fig. 1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. Authors are from Trillion Labs and KAIST AI, but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Trillion Labs and KAIST AI, with correspondence emails provided."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Trillion Labs is a company (authors Han, Shin are affiliated) that could have financial interest in efficient pre-training methods."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Authors from Trillion Labs may have financial interests related to efficient LLM training methods."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for the OLMo models or GPT-4o used. The paper does not discuss when the pre-training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap between OLMo pre-training data and benchmark test sets (GSM8K, MATH500, ARC-C, etc.)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Benchmarks like ARC-C (2018), GSM8K (2021), MATH500 (2021) were published years before model training, creating contamination risk. This is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cost is discussed: 'a small one-time cost of under $10 per benchmark' for generating Rϕ (§5), compute measured in FLOPs throughout (Fig. 6), and 'thousands of H100 hours' for additional training runs (§4.1(iii))."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Compute is quantified via FLOPs formula (6ND, §4.1(i)), hardware specified (256 H100 GPUs, Appendix C.1), and compute savings factors explicitly stated (100.2× to 733.4×)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Experiment (i) uses results 'averaged across three pre-training seeds' (§4.1(i), following Magnusson et al. 2025). 5-fold cross-validation provides variance estimates for experiments (ii-iii)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Three pre-training seeds for experiment (i) (§4.1(i)), 5-fold cross-validation for experiments (ii-iii) (§4.1(ii))."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "The curve fitting hypothesis space is defined a priori: 'linear, quadratic, exponential, and logarithmic. This hypothesis space was defined a priori to avoid overfitting' (§4.1(ii))."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Configuration selection is transparent: 'Curve fitting selects the best function based on train R²' (§4.1(ii)), with the hypothesis space defined before experiments."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No multiple comparison correction is applied despite comparing 7 methods across 6 benchmarks in multiple experimental settings."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of author-evaluation bias. The authors implemented all baselines and their own method without acknowledging this potential bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Fig. 6 explicitly plots Decision Accuracy as a function of FLOPs, and Fig. 7a compares RBRIDGE at 1B against larger proxy models (7B, 13B) using the target metric."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the six benchmarks actually measure 'reasoning' as claimed. The paper assumes benchmarks like ARC-C and CQA measure reasoning without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. RBRIDGE is a metric computation method, not an agent or scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Benchmarks like GSM8K (2021) and MATH500 (2021) predate the OLMo training data, and this is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The 5-shot CoT evaluation setup provides exemplars that could leak information, but this is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples share structural similarities with pre-training data or with each other."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "RBRIDGE reduces dataset ranking compute cost by over 100× relative to the best baseline.",
       "evidence": "Fig. 6b shows RBRIDGE achieves equivalent Decision Accuracy with 100.2× to 733.4× fewer FLOPs across 25 pre-training datasets (§4.2(i)).",
       "supported": "strong"
     },
     {
       "claim": "RBRIDGE achieves the strongest correlation across six reasoning benchmarks at 1B to 32B scale.",
       "evidence": "Tab. 2 shows RBRIDGE achieves best average train R² (0.874) and test MAE (1.384) for 1B→13B, and best averages for 1B→13B+SFT and 1B→32B across all methods.",
       "supported": "strong"
     },
     {
       "claim": "RBRIDGE outperforms proxy models 7-13× larger using the target metric.",
       "evidence": "Fig. 7a shows RBRIDGE at 1B achieves lower test MAE than Acc./p@1 at 7B and 13B scale for predicting 32B performance.",
       "supported": "strong"
     },
     {
       "claim": "RBRIDGE enables zero-shot functional relationship transfer across pre-training datasets.",
       "evidence": "Tab. 3 shows transferred function achieves 5/5 correct rankings and MAE of 0.043-1.417 on most benchmarks (one outlier at 9.716), at 1B→7B scale.",
       "supported": "moderate"
     },
     {
       "claim": "Reasoning trace Rϕ is more in-distribution than benchmark-provided gold labels.",
       "evidence": "Fig. 4 shows 74.7% average NLL decline when using Rϕ across five reasoning benchmarks, and Tab. 1 shows better proxy performance with Rϕ.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "The paper evaluates OLMo models on public benchmarks (GSM8K, MATH500, ARC-C, HumanEval) without any discussion of data contamination, despite these benchmarks being publicly available before model training."
     },
     {
       "flag": "Limited zero-shot transfer evaluation",
       "detail": "The zero-shot transfer result (experiment iii) is demonstrated on only one additional dataset at 1B→7B scale with a single data point (1T tokens), making the claim of 'zero-shot transfer' rest on very limited evidence."
     },
     {
       "flag": "No significance testing",
       "detail": "All comparisons between RBRIDGE and baselines rely on point comparisons of R² and MAE values without any statistical significance tests, despite running multiple comparisons across methods and benchmarks."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational work on scaling laws for predicting LLM performance, which RBRIDGE extends to reasoning tasks."
     },
     {
       "title": "Emergent abilities of large language models",
       "authors": ["Jason Wei", "Yi Tay", "Rishi Bommasani"],
       "year": 2022,
       "relevance": "Defines the emergence phenomenon that RBRIDGE aims to bridge — reasoning capabilities appearing only at larger model scales."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Proposes continuous metrics (TED, MPCA) as baselines, arguing emergence is a metric artifact. RBRIDGE compares against these."
     },
     {
       "title": "DataDecide: How to predict best pretraining data with small experiments",
       "authors": ["Ian Magnusson", "Nguyen Tai", "Ben Bogin"],
       "year": 2025,
       "relevance": "Provides the dataset ranking benchmark and protocol used in RBRIDGE's experiment (i)."
     },
     {
       "title": "Predicting emergent capabilities by finetuning",
       "authors": ["Charlie Victor Snell", "Eric Wallace", "Dan Klein"],
       "year": 2024,
       "relevance": "Proposes intermediate SFT to predict emergent capabilities, used as a baseline (iSFT) in RBRIDGE experiments."
     },
     {
       "title": "An empirical analysis of compute-optimal large language model training",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch"],
       "year": 2022,
       "relevance": "Chinchilla scaling laws for compute-optimal training — foundational work RBRIDGE builds upon for cost-efficient pre-training."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Introduces chain-of-thought reasoning traces used as gold labels in RBRIDGE."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark used in RBRIDGE experiments for evaluating code generation capability."
     },
     {
       "title": "Understanding emergent abilities of language models from the loss perspective",
       "authors": ["Zhengxiao Du", "Aohan Zeng", "Yuxiao Dong"],
       "year": 2024,
       "relevance": "Studies emergence at granular scale (300M-3B), directly motivating RBRIDGE's approach to bridging scale gaps."
     },
     {
       "title": "DoReMi: Optimizing data mixtures speeds up language model pretraining",
       "authors": ["Sang Michael Xie", "Hieu Pham", "Xuanyi Dong"],
       "year": 2023,
       "relevance": "Uses proxy models for data mixture optimization — related approach that RBRIDGE improves upon for reasoning tasks."
     }
   ]
 }