ai-research-survey

scan-v5.json (27579B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
     "authors": [
       "DeepSeek-AI"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2501.12948",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All major abstract claims are supported: pure RL enabling reasoning is demonstrated by R1-Zero (Figure 1, Table 8), emergent self-reflection is shown in Figure 9 and Table 2, superior performance to SFT counterparts is confirmed in Table 8, and distillation enabling smaller models is shown in Table 15.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The R1-Zero experiment directly tests pure RL from a base model with no SFT; staged ablations (Dev1–Dev3 in Table 3) isolate component contributions; language consistency reward is ablated (Figure 7); distillation vs. pure RL is compared (Table 16). Multiple causal claims are supported by appropriate ablation designs.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states results are strongest for 'verifiable tasks such as mathematics, coding competitions, and STEM fields'; Section 6 limitations acknowledge degraded performance for open-ended writing, software engineering, and non-Chinese/English languages.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes all improvements to RL but does not discuss alternative explanations: whether gains stem from longer output generation, additional training compute, superior base model quality, or reward function design rather than the RL mechanism per se.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'reasoning capability' but measures benchmark accuracy (AIME, MATH-500, LiveCodeBench) without explicitly discussing the relationship between benchmark performance and the broader construct of reasoning ability.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Conclusion, Limitation, and Future Work' contains a dedicated, multi-paragraph limitations section listing specific capability gaps beyond a single sentence.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are identified: prompting sensitivity (few-shot consistently degrades performance), reward hacking documented with example (Figure 6), language mixing in non-Chinese/English queries, and limited RL for software engineering tasks due to long evaluation times.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states the approach is limited to tasks with reliable verifiers and notes that 'for complex tasks that cannot be effectively evaluated by a reliable reward model, scaling up pure RL methods remains an open challenge.'",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgment section is present. DeepSeek-AI appears to be corporate self-funded but this is not explicitly stated.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are from DeepSeek-AI, clearly stated as the sole author affiliation with contact email research@deepseek.com.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "DeepSeek-AI employees are evaluating their own model and comparing it favorably against competitors; the funder (DeepSeek) is not independent of the outcome.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patents, equity, or financial interests declaration appears anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key technical terms are formally defined: GRPO is specified with full equations (1–3), reward design (accuracy + format rewards) is explained with formula (4), cold start data is described with examples, and the multi-stage pipeline is diagrammed in Figure 2.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states its contribution: showing that LLM reasoning can be incentivized through pure RL without human-labeled demonstrations, producing models that match OpenAI-o1 on reasoning benchmarks.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section H provides substantive related work covering chain-of-thought (Wei et al. 2022), inference-time scaling, and RL for reasoning, explicitly contrasting their approach with PRM, MCTS, STaR, and RLHF methods.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Inference code is released at https://github.com/deepseek-ai/DeepSeek-V3 with torchrun commands and model weights on HuggingFace at https://huggingface.co/deepseek-ai under MIT license.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "All evaluation benchmarks (AIME, MATH-500, MMLU, LiveCodeBench, etc.) are standard public benchmarks used unmodified; training data release is promised but URL is placeholder 'xxx' in the paper.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": true,
           "justification": "The inference code example includes 'pip install -r requirements.txt', referencing a requirements file in the GitHub repository; training infrastructure specifies H800 GPUs, vLLM, and DualPipe algorithm.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Inference instructions are provided (torchrun commands), but step-by-step training reproduction is not feasible: training data URL is placeholder 'xxx', and the full RL pipeline requires 147K GPU hours of H800 compute not documented in reproducible form.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All main result tables report point estimates only; no confidence intervals or error bars are shown despite results being averaged over k=4–64 samples per question.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": true,
           "justification": "Table captions state 'Numbers in bold denote the performance is statistically significant (t-test with p < 0.01)', applied to comparative performance claims.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Raw performance numbers are reported throughout enabling direct effect size interpretation (e.g., AIME 79.8% vs 79.2% for o1, MATH-500 97.3% vs 96.4%).",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The paper states k values (k=64 for AIME, k=16 for MATH, k=8 for LCB) but provides no power analysis or formal justification for these specific choices.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No standard deviations or variance across evaluation runs are reported in any of the main results tables, despite averaging over multiple samples.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Comprehensive baselines in Table 8: Claude-3.5-Sonnet-1022, GPT-4o-0513, DeepSeek-V3, OpenAI-o1-mini, OpenAI-o1-1217.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines are frontier models as of early 2025: OpenAI-o1-1217 (December 2024), Claude-3.5-Sonnet-1022, and GPT-4o-0513 — the best available comparators.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Multiple ablations: stage-by-stage comparison (R1-Zero, Dev1–Dev3, R1 in Table 3), language consistency reward ablation (Figure 7), and distillation vs. large-scale RL comparison (Table 16).",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Over 20 benchmarks used spanning math (AIME 2024, MATH-500, CNMO), code (LiveCodeBench, Codeforces, SWE-Bench, Aider), knowledge (MMLU, GPQA Diamond), and instruction following (IFEval, AlpacaEval 2.0, ArenaHard).",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "ChatbotArena crowdsourced pairwise human preference evaluation is used (Figures 11–12), showing DeepSeek-R1 ranking first alongside OpenAI-o1 on the style control setting.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "All evaluation benchmarks are held-out test sets; additionally AIME 2025 (released after training cutoff) is used to assess generalization to genuinely unseen problems.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Per-category breakdowns provided: MMLU by subject (Figure 15–16), math by competition category (Figure 17), LiveCodeBench by difficulty (Table 14), and safety by category (Tables 9–11).",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Failure cases are shown: reward hacking during training (Figure 6), language mixing in multilingual queries, overthinking on simple problems, and Section G.2 explicitly reports failed PRM and MCTS approaches.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section G.2 'Unsuccessful Attempts' dedicates a full section to reporting failures with Process Reward Models (annotation difficulty, reward hacking) and Monte Carlo Tree Search (exponential search space, value model difficulties).",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Base model is DeepSeek-V3-Base; all baseline models include version dates (Claude-Sonnet-3.5-1022, GPT-4o-0513, OpenAI-o1-1217); intermediate checkpoints (Dev1–Dev3) are labeled.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Multiple prompts provided in full: R1-Zero training template (Table 1), reward model prompt (Listing 8), SFT trajectory examples (Listings 5–7), test case generation prompts (Listing 2), and benchmark evaluation prompts (Tables 18–32).",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Full hyperparameters reported in Appendix B.4: learning rate (3e-6), KL coefficient (0.001), clip ratio (ε=10), sampling temperature (1.0), batch size (512), max sequence lengths (32,768–65,536 tokens), per-model distillation learning rates in Table 6.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "RL infrastructure described in detail (Figure 5, Appendix B.1): four distinct modules (rollout via vLLM, inference, rule-based reward, training), expert parallelism strategy, VRAM management, data packing strategy.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Data preprocessing documented: 10-gram decontamination (removing ~6M math texts), cold start data generation pipeline with rejection sampling and human refinement, SFT filtering (language mixing, length, repetition detection), evaluation prompt formats.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "All evaluation benchmarks (AIME, MATH-500, LiveCodeBench, MMLU, etc.) are publicly accessible; model weights are available on HuggingFace enabling independent evaluation replication.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Data collection documented in Appendix B.3 and Table 4: 26K math, 17K code, 22K STEM, 15K logic, 66K general prompts with sources, formats, average lengths, and construction procedures.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants were recruited; standard public benchmarks were used for evaluation.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Full data pipeline documented from collection through SFT: cold start generation (Listings 1–3), rejection sampling, human annotation and verification steps, 800K SFT data statistics (Table 5), and decontamination procedures.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": true,
           "justification": "Appendix D.1 explicitly states 'DeepSeek-V3 base has a knowledge cutoff date of July 2024.'",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix D.1 'Decontamination' explicitly discusses overlap: 10-gram filtering removed ~6M math-related texts; post-training data sourced exclusively from pre-2023 competitions; paper acknowledges n-gram filtering cannot prevent paraphrase contamination.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": true,
           "justification": "AIME 2025 (post-July 2024 cutoff) is used to test generalization to genuinely unseen problems (Table 13), showing 75% solve rate approaching o1's 80%.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects research in this study.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants requiring IRB approval.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "Per-query inference cost and latency are not reported; only training costs appear in Table 7.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Table 7 provides detailed training costs: R1-Zero 101K H800 GPU hours ($202K), SFT data creation 5K hours ($10K), DeepSeek-R1 41K hours ($82K), total 147K GPU hours ($294K at $2/GPU-hour).",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "DeepSeek-R1-Zero achieves 79.8% pass@1 on AIME 2024 via pure RL without any supervised fine-tuning",
       "evidence": "Figure 1 shows training progression from 15.6% to 77.9% on AIME 2024; Table 8 reports final 79.8% pass@1 and 86.7% with cons@64 self-consistency",
       "supported": "strong"
     },
     {
       "claim": "DeepSeek-R1 performance matches OpenAI-o1-1217 on mathematical reasoning benchmarks",
       "evidence": "Table 8: DeepSeek-R1 79.8% vs o1 79.2% on AIME 2024, 97.3% vs 96.4% on MATH-500, 78.8% vs unreported on CNMO 2024",
       "supported": "strong"
     },
     {
       "claim": "Advanced reasoning behaviors (self-reflection, verification, 'aha moments') emerge spontaneously from RL training without explicit instruction",
       "evidence": "Figure 9 shows 5–7x increase in reflective word frequency during training; Table 2 shows the model spontaneously generating 'Wait, wait. Wait. That's an aha moment' to self-correct",
       "supported": "moderate"
     },
     {
       "claim": "Distilled small models (1.5B–70B) substantially outperform non-reasoning models of comparable or larger size",
       "evidence": "Table 15: DeepSeek-R1-Distill-Qwen-1.5B achieves 28.9% AIME 2024 pass@1, surpassing GPT-4o-0513 (9.3%) and Claude-3.5-Sonnet (16.0%)",
       "supported": "strong"
     },
     {
       "claim": "Larger base model capacity is a prerequisite for RL-induced reasoning improvements to emerge",
       "evidence": "Section G.1 reports that 7B dense and 16B MoE models showed no meaningful AIME improvements under RL, while 32B+ models showed substantial gains",
       "supported": "moderate"
     },
     {
       "claim": "Distillation from a strong reasoning model outperforms training smaller models with large-scale RL directly",
       "evidence": "Table 16: DeepSeek-R1-Distill-Qwen-32B (72.6% AIME) substantially outperforms Qwen2.5-32B-Zero trained with 10K RL steps (47.0% AIME)",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "empirical"
   ],
   "key_findings": "DeepSeek-R1-Zero demonstrates that pure reinforcement learning applied to a capable base model can autonomously develop sophisticated reasoning behaviors—self-reflection, verification, dynamic strategy adaptation—without any human-annotated demonstrations, reaching 79.8% on AIME 2024 and matching OpenAI-o1. The multi-stage DeepSeek-R1 pipeline (cold start + RL + SFT + RL) addresses readability and language consistency issues while maintaining frontier reasoning performance. Knowledge distillation from R1 into small models (1.5B–70B) produces models that dramatically outperform non-reasoning models of similar size. Two key negative findings: process reward models and MCTS were attempted and abandoned due to reward hacking and scaling difficulties; and smaller base models (7B, 16B MoE) failed to benefit from RL, establishing model scale as a prerequisite.",
   "red_flags": [
     {
       "flag": "Training data URL placeholder",
       "detail": "The paper states SFT and RL training data is released 'at xxx' — a literal placeholder, meaning training data was not actually accessible at publication time, preventing training reproduction."
     },
     {
       "flag": "Self-evaluation with no independent replication",
       "detail": "DeepSeek-AI employees evaluate their own model; results for OpenAI-o1-1217 are taken from official reports rather than independently measured, making direct comparisons unverifiable."
     },
     {
       "flag": "Severe jailbreak vulnerability",
       "detail": "Table 11 shows DeepSeek-R1 without risk control reaches 85.9% unsafe rate under jailbreak attacks — the highest of all tested models. The paper acknowledges enhanced reasoning makes dangerous content more operationally feasible."
     },
     {
       "flag": "No variance reporting",
       "detail": "All main benchmark tables report only point estimates; no standard deviations or confidence intervals are shown despite results being averaged over k=4–64 samples per question."
     },
     {
       "flag": "Alternative explanations unaddressed",
       "detail": "Improvements attributed solely to RL without considering confounders: longer output generation, additional training compute (147K GPU-hours), or base model quality (DeepSeek-V3-Base already strong) could partially explain gains."
     }
   ],
   "cited_papers": [
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "relevance": "Foundational CoT work that R1 extends via RL; the primary paradigm R1 challenges by showing RL can discover reasoning without human-curated demonstrations"
     },
     {
       "title": "Training language models to follow instructions with human feedback (InstructGPT)",
       "relevance": "Establishes the SFT+RLHF paradigm that R1 partially circumvents; key baseline for comparing post-training approaches"
     },
     {
       "title": "DeepSeekMath: Pushing the limits of mathematical reasoning in open language models",
       "relevance": "Introduces GRPO algorithm used as R1's RL backbone; directly cited as the training algorithm"
     },
     {
       "title": "Let's verify step by step",
       "relevance": "Process reward model work that DeepSeek-R1 attempted and abandoned (Section G.2), providing important negative results context for the field"
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "relevance": "Self-consistency decoding (cons@16, cons@64) is used in evaluating R1-Zero and boosts AIME accuracy from 79.8% to 86.7%"
     },
     {
       "title": "STaR: Bootstrapping reasoning with reasoning",
       "relevance": "Prior RL-based reasoning enhancement that R1 builds upon; key comparison point for showing R1's approach differs by starting from pure RL on base models"
     },
     {
       "title": "DeepSeek-V3 technical report",
       "relevance": "DeepSeek-V3-Base is the base model for all R1 variants; understanding the base model is essential for interpreting what RL adds"
     },
     {
       "title": "Scaling LLM test-time compute optimally can be more effective than scaling parameters for reasoning",
       "relevance": "Related work on test-time compute scaling; R1's adaptive CoT length is analyzed in relation to this paradigm in Section E.4"
     },
     {
       "title": "Proximal policy optimization algorithms",
       "relevance": "PPO is the primary RL baseline compared against GRPO (Figure 4, Appendix A.3); establishing why GRPO is preferred for large-scale training"
     },
     {
       "title": "Language models are few-shot learners (GPT-3)",
       "relevance": "Establishes emergent capabilities framework used to contextualize R1's emergent reasoning behaviors"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Model weights freely available on HuggingFace under MIT license; practitioners can immediately use distilled 1.5B–70B models for math, code, and reasoning tasks."
     },
     "surprise_contrarian": {
       "score": 3,
       "justification": "Demonstrating that pure RL without SFT produces frontier reasoning challenged the field's consensus that extensive human demonstrations were essential for capable post-training."
     },
     "fear_safety": {
       "score": 2,
       "justification": "The paper documents 85.9% unsafe rate under jailbreak attacks without risk control and explicitly notes enhanced reasoning makes dangerous content more operationally feasible."
     },
     "drama_conflict": {
       "score": 3,
       "justification": "Directly matches OpenAI-o1 on math benchmarks at $294K training cost under MIT license, challenging the assumption that frontier reasoning models require closed proprietary development."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Model weights downloadable from HuggingFace immediately; distilled versions (1.5B–70B) accessible on consumer hardware; official API available."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "DeepSeek-R1 became one of the most discussed AI papers of early 2025, generating 1,351 HN points and triggering significant market reactions upon release."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "42823568",
         "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via RL",
         "points": 1351,
         "comments": 1056,
         "url": "https://news.ycombinator.com/item?id=42823568",
         "created_at": "2025-01-25T18:39:49Z"
       },
       {
         "hn_id": "42915646",
         "title": "Stack Overflow Meets Replication: Security Research Amid Evolving Code Snippets",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42915646",
         "created_at": "2025-02-03T06:49:46Z"
       }
     ],
     "top_points": 1351,
     "total_points": 1352,
     "total_comments": 1056
   }
 }