ai-research-survey

scan-v5.json (26668B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "LLMs Encode Their Failures: Predicting Success from Pre-Generation Activations",
     "authors": [
       "William Lugoloobi",
       "Thomas Foster",
       "William Bankes",
       "Chris Russell"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.09924",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All major claims (difficulty divergence, probe outperformance, 70% cost reduction) are directly supported by Tables 1-2, Figure 1, and Figures 3-4 with numerical evidence.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Mechanistic claims about what models 'encode' are inferred from probe correlations, not interventional. Routing claims are demonstrated but causality not proven—no ablations on what information probes actually use.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Testing spans 5 benchmarks (E2H-AMC, MATH, GSM8K, AIME, LiveCodeBench), 4 model families. Paper explicitly bounds scope: 'We do not study cross-domain or cross-dataset probe transfer... leaving open questions about generalization.'",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "One mechanism proposed (output length correlates with human difficulty) but alternatives not systematically considered. E.g., could probes just be picking up on question complexity proxies rather than true success likelihood?",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Clear distinction between human IRT difficulty, expected success rate, and binary policy success. Paper states 'model-specific difficulty provides a more reliable predictor than human difficulty' with explicit measurement definitions.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 explicitly titled 'CONCLUSION, LIMITATIONS AND FUTURE WORK' with dedicated limitations subsection.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats: 'Probe performance degrades under extended reasoning'; 'We focus on linear probes at single post-instruction position'; 'We do not study cross-domain probe transfer'; 'routing policies are intentionally simple.'",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly states boundaries: no non-linear probes, no cross-dataset transfer, no adaptive routing, no vision/multimodal domains. Future work section delineates what remains open.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "Preprint lists no funding statement or acknowledgments. No source of financial support mentioned.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors list Oxford Internet Institute, FLAIR/Oxford, and UCL affiliations. No company relationships with evaluated models disclosed.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funder disclosed, so independence cannot be assessed. Mark as NA—no financial stake evident.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. No mention of patents, equity stakes, or consulting relationships with OpenAI, Anthropic, or other model providers.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms precisely defined: linear probes trained on activations; pre-generation activations as 'residual stream activations (pre-layer norm)' at 'post-instruction positions'; policy-specific success under Maj@K and greedy decoding.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "'Main Contributions' section explicitly lists three claims: (1) Human vs model difficulty are distinct; (2) Probes generalize across settings; (3) Routing achieves 70% cost reduction. Contribution to efficiency + interpretability is clear.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 engages with 4 research lines: correctness prediction (Kadavath, Burns, Cencerrado), difficulty estimation (Lugoloobi & Russell, Lee et al.), test-time compute (Wang et al., DeepSeek-R1), and routing (Chen et al., Ding et al.). Positions contributions relative to each.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "GitHub repo provided: 'Our code is available at: https://github.com/KabakaWilliam/llms_know_difficulty'.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "All benchmarks are public: E2H-AMC (Ding et al. 2024), MATH (Cobbe et al. 2021), GSM8K (Hendrycks et al. 2021), AIME (public), LiveCodeBench (Veeraboina 2023). No custom private datasets.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Appendix 5.3 specifies VLLM + model hyperparameters (temperature, max_length). Missing: Python version, VLLM version number, PyTorch version, requirements.txt or environment file.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Code on GitHub presumably includes instructions, but the paper lacks step-by-step reproduction guide. Appendix 5.1 describes probe training (Ridge/logistic regression) but not end-to-end pipeline with data loading, preprocessing, probe extraction.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Tables 1-2, 4-5 report only point estimates (Spearman ρ, AUROC, accuracy). Figures 1-4 have no error bars. No confidence intervals reported despite multiple benchmarks and models.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No p-values, t-tests, or significance tests comparing probes to baselines. Claims like 'substantially outperforming' are made without statistical backing.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Spearman ρ (0.83-0.87), AUROC (0.76-0.91), and cost reduction percentages (17-70%) serve as effect sizes. Table 2 shows differences like 0.84 vs 0.64 AUROC.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "E2H-AMC: 4k problems stated. MATH, GSM8K, AIME, LiveCodeBench sizes mentioned but not all. No power analysis or justification for sample size choices provided.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No std dev, variance, or run-to-run noise reported. Results appear to be single-run point estimates. This is a significant gap for model evaluation.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "TF-IDF and question length as text baselines. Random and oracle routing as routing baselines. Single-model baselines (Qwen, DeepSeek, GPT-OSS variants).",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Model baselines are recent: Qwen2.5 (2024/2025), DeepSeek-R1 (2025), GPT-OSS-20B (recent). TF-IDF is old but acceptable as text feature baseline.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": false,
           "justification": "No ablation removing components (e.g., different layer positions, different activation features). Paper does sensitivity analysis (greedy vs Maj@K, reasoning modes) but no component removal.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Spearman ρ for difficulty ranking, AUROC for binary classification, cost-accuracy Pareto curves for routing. Multiple angles on performance.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable—no human evaluation of model outputs. E2H-AMC uses existing human IRT scores but paper does not conduct new human studies.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "'80/20 train–validation split for hyperparameter selection, and report our best probe results on a held-out test set.' Standard practice followed.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results broken down by dataset (MATH, GSM8K, AIME, LiveCodeBench), model (Qwen, DeepSeek, GPT-OSS), reasoning mode (low/medium/high), and decoding (greedy vs Maj@K).",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Paper identifies probe quality degradation under extended reasoning and oracle gap but does not show specific failure examples or analyze patterns (e.g., which question types cause probe failures?).",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Explicitly reports: 'probe AUROC drops monotonically as reasoning budgets increase, even as accuracy improves' (0.78→0.64). Gap to oracle shown in Figures 3-4.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Models referred to as 'Qwen2.5-Math-7B', 'GPT-OSS-20B', 'DeepSeek-R1-Distill-Qwen-7B' with no snapshot dates or commit hashes. Exact versions cannot be reproduced.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "No actual prompts shown. Paper mentions 'model's chat template' and post-instruction positions but does not display the system prompt or query formatting used.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "Table 6 reports temperature, max_length, k for rollouts. Appendix 5.1: regularization α tuned over range. Missing: learning rate, optimizer, batch size, number of epochs for probe training.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "N/A—no agentic scaffolding. Paper evaluates base model behavior without chains-of-thought setup beyond what models generate naturally.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": false,
           "justification": "Basic info: LiveCodeBench uses 'contamination-aware temporal splits'; benchmarks are standard. Missing: answer parsing code, how incorrect/malformed responses handled, tokenization details.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "E2H-AMC, MATH, GSM8K, AIME, LiveCodeBench all publicly available. Model rollouts and probe outputs derivable from released code.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "E2H-AMC: human IRT scores from psychometric calibration on student performance (Ding et al. 2024). Model responses: K=50 rollouts at T=1 for math benchmarks, specified in Table 6.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "N/A—no human recruitment. Paper uses existing human-labeled benchmark (E2H-AMC) with no new participant recruitment.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": false,
           "justification": "Pipeline described: benchmarks → model rollouts → activation extraction at post-instruction → probe training on 80% → evaluation on 20%. But missing details on data loading, batch processing, error handling.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training data cutoff dates provided for any model. 'Qwen2.5-Math-7B', 'gpt-oss-20b', 'DeepSeek-R1' dates unknown. Critical given 2024-2025 AIME/LiveCodeBench exposure risk.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "LiveCodeBench: 'contamination-aware temporal splits' used. But MATH, GSM8K, AIME: no discussion of potential overlap. Known literature flags MATH/GSM8K as contaminated in some model training; not addressed here.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Only LiveCodeBench addressed with temporal splits. E2H-AMC, MATH, GSM8K, AIME contamination not discussed despite potential 2024-2025 training data overlap.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Linear probes on pre-generation activations can predict model success with AUROC > 0.7 across most inference settings",
       "evidence": "Table 2: AUROC values 0.76–0.91 for Maj@5 predictions on Qwen and DeepSeek models; GPT-OSS-20B shows 0.78 (low) to 0.64 (high reasoning) range.",
       "supported": "strong"
     },
     {
       "claim": "Human difficulty and model difficulty are distinct and encoded separately in pre-generation activations",
       "evidence": "Table 1: Human IRT Spearman ρ = 0.83–0.87 vs model success rate ρ = 0.40–0.64 from identical activations. Figure 1A shows alignment between predicted difficulties diverges with reasoning.",
       "supported": "strong"
     },
     {
       "claim": "Model-derived difficulty signals are more predictive of actual performance than human difficulty signals",
       "evidence": "Figure 1B: Probe-predicted model difficulty consistently outperforms human difficulty for predicting Maj@5 failure across all reasoning modes, despite divergence.",
       "supported": "strong"
     },
     {
       "claim": "Probe quality degrades monotonically as extended reasoning increases, despite improved task accuracy",
       "evidence": "Table 2 & 4: GPT-OSS-20B Maj@5 AUROC drops 0.78→0.64 as reasoning increases (low→high); accuracy improves 86.6%→92.0% in same setting.",
       "supported": "strong"
     },
     {
       "claim": "Probe-guided routing can match best-performing model accuracy while reducing inference cost by up to 70% on MATH",
       "evidence": "Figure 3 (right): Utility router achieves GPT-OSS-20B-high accuracy (92%) at $15 vs $28 baseline (70% cost reduction). Table 8 confirms results.",
       "supported": "strong"
     },
     {
       "claim": "Linear probes substantially outperform text-based baselines (TF-IDF, question length) for predicting success",
       "evidence": "Table 1-2: Linear probe ρ=0.83–0.87 vs TF-IDF ρ=0.72 for human difficulty; AUROC 0.84 vs 0.64 for model success. Across all settings, probes dominate baselines.",
       "supported": "strong"
     },
     {
       "claim": "With extended reasoning, models spend more tokens on problems humans find difficult even when those problems are within model competence",
       "evidence": "Figure 2: Chain-of-thought length positively correlated with human IRT difficulty across all reasoning modes, but negatively correlated with empirical success and probe-predicted success.",
       "supported": "strong"
     },
     {
       "claim": "Routing effectiveness is bottlenecked by probe reliability (AUROC), not routing algorithm sophistication",
       "evidence": "Figures 3-4 and oracle analysis: Gap to oracle (perfect knowledge) narrows when probe quality is high (AIME) and widens when probe quality degrades (GPT-OSS-20B high reasoning). Simple utility rule achieves near-oracle when AUROC > 0.8.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "empirical",
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "Linear probes extract success predictions from LLM pre-generation activations with >70% AUROC, substantially outperforming text features. Crucially, models encode a model-specific difficulty signal distinct from human difficulty (ρ=0.40–0.64 vs ρ=0.83–0.87), and this divergence increases with extended reasoning. Probe-guided routing matches top-performing models at 17–70% cost reduction on math benchmarks, demonstrating practical efficiency gains; however, probe quality degrades sharply under deep reasoning, suggesting that extended chain-of-thought reasoning encodes difficulty information in ways that are not linearly accessible at the pre-generation stage.",
   "red_flags": [
     {
       "flag": "No confidence intervals or significance testing",
       "detail": "All results report single point estimates (Spearman ρ, AUROC, accuracy) with no error bars, confidence intervals, or p-values. Unclear if observed differences between methods (e.g., probe vs TF-IDF) are statistically significant."
     },
     {
       "flag": "Model versions not precisely specified",
       "detail": "Marketing names ('Qwen2.5-Math-7B', 'GPT-OSS-20B') lack snapshot dates or commit hashes. Exact reproduction of setup impossible; models may have been updated since paper submission."
     },
     {
       "flag": "Actual prompts not provided",
       "detail": "Paper mentions 'chat template' and post-instruction positions but does not display system prompts or query formatting. Reproducibility limited without prompt details."
     },
     {
       "flag": "Training cutoff dates not stated",
       "detail": "No training data cutoff provided for any model evaluated. Given 2024-2025 AIME/LiveCodeBench exposure, potential train-test contamination cannot be ruled out, especially for MATH/GSM8K."
     },
     {
       "flag": "No variance or run-to-run noise reported",
       "detail": "All metrics appear to be single-run point estimates. No discussion of whether random seed, batch order, or other sources of variance affect probe training or routing results."
     },
     {
       "flag": "Mechanistic explanations are correlational, not causal",
       "detail": "Paper infers what models 'encode' from probe correlation with outcomes, but no interventional evidence (e.g., ablating components, perturbing activations) proves causality. Alternative explanations (e.g., probes detecting output length, not true difficulty) not ruled out."
     },
     {
       "flag": "Cross-domain generalization not tested",
       "detail": "Probes trained on math do not show transfer to code or vice versa. Limits practical applicability—would need separate probes per domain/task."
     },
     {
       "flag": "Probe degradation under extended reasoning limits utility",
       "detail": "AUROC drops from 0.78 to 0.64 as reasoning increases, exactly when routing matters most. Paper acknowledges but does not solve: 'probe reliability becomes the bottleneck' for advanced reasoning models."
     }
   ],
   "cited_papers": [
     {
       "title": "Language Models (Mostly) Know What They Know",
       "authors": "Kadavath et al.",
       "year": 2022,
       "relevance": "Foundational work on model self-assessment and confidence; establishes that models can evaluate their own correctness."
     },
     {
       "title": "The Internal State of an LLM Knows When It's Lying",
       "authors": "Azaria & Mitchell",
       "year": 2023,
       "relevance": "Demonstrates extractable 'truthfulness directions' in internal activations; directly related approach to probing latent knowledge."
     },
     {
       "title": "Discovering Latent Knowledge in Language Models Without Supervision",
       "authors": "Burns et al.",
       "year": 2024,
       "relevance": "Unsupervised extraction of correctness signals from activations; comparison point for supervised probe approach."
     },
     {
       "title": "No Answer Needed: Predicting LLM Answer Accuracy from Question-Only Linear Probes",
       "authors": "Cencerrado et al.",
       "year": 2025,
       "relevance": "Directly related work on predicting correctness from activations; weaker performance on math (AUROC ≈0.6–0.7) vs this paper's 0.78–0.84."
     },
     {
       "title": "LLMs Encode How Difficult Problems Are",
       "authors": "Lugoloobi & Russell",
       "year": 2025,
       "relevance": "Prior work showing difficulty is linearly decodable; this paper extends to human vs model difficulty distinction and routing applications."
     },
     {
       "title": "Probing the Difficulty Perception Mechanism of Large Language Models",
       "authors": "Lee et al.",
       "year": 2025,
       "relevance": "Difficulty perception mechanisms in LLMs; paper adds comparison with human IRT difficulty and extends to routing."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": "Chen et al.",
       "year": 2024,
       "relevance": "Model routing for cost-accuracy tradeoffs; establishes motivation and prior baselines for routing research."
     },
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "authors": "Ding et al.",
       "year": 2023,
       "relevance": "Heuristic-based routing using input complexity; this paper replaces heuristics with learned difficulty probes."
     },
     {
       "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models",
       "authors": "Wang et al.",
       "year": 2022,
       "relevance": "Majority voting aggregation (Maj@K) for test-time scaling; paper analyzes probe robustness across sampling policies."
     },
     {
       "title": "Easy2Hard-Bench: Standardized Difficulty Labels for Profiling LLM Performance and Generalization",
       "authors": "Ding et al.",
       "year": 2024,
       "relevance": "Provides E2H-AMC benchmark with human IRT difficulty labels; critical resource enabling human vs model difficulty comparison."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Routing shows 17–70% cost reduction on benchmarks, but probe degradation under extended reasoning limits real-world utility for advanced models. Unclear if gains transfer to complex multi-step inference pipelines."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Divergence between human and model difficulty is interesting. Finding that probes degrade with extended reasoning contradicts intuition that better reasoning enables better self-assessment, but effect is documented not explained."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Some interpretability value (understanding internal representations), but no direct safety implications. Does not raise new risk concerns or address alignment/deception."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Straightforward empirical technical work. No controversial claims, stakeholder conflict, or dramatic findings. Well-executed but incremental contribution."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub, but reproducing full pipeline requires 50 rollouts across multiple benchmarks (expensive compute). Feasible to demo on small subsets or use pre-computed results."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from Oxford Internet Institute, FLAIR, UCL—solid academic institutions but no industry prestige or famous lab association. Preprint with modest HN engagement (1 point, 3 comments)."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "46995551",
         "title": "Routing LLM queries using internal success predictions (70% cost reduction)",
         "points": 1,
         "comments": 3,
         "url": "https://news.ycombinator.com/item?id=46995551",
         "created_at": "2026-02-12T21:33:47Z"
       }
     ],
     "top_points": 1,
     "total_points": 1,
     "total_comments": 3
   }
 }