ai-research-survey

scan.json (31505B)

---

 {
   "paper": {
     "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
     "authors": [
       "DeepSeek-AI"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2501.12948"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "DeepSeek-R1-Zero demonstrates that pure RL without supervised fine-tuning can incentivize sophisticated reasoning behaviors (self-reflection, verification) in LLMs, achieving 79.8% on AIME 2024 and 97.3% on MATH-500, matching OpenAI-o1. The multi-stage pipeline (cold-start SFT → RL → rejection sampling SFT → RL) produces DeepSeek-R1, which ranks alongside top closed-source models on ChatbotArena. Distilled smaller models (1.5B-70B) outperform GPT-4o and Claude-3.5-Sonnet on math benchmarks, and distillation outperforms RL alone on smaller architectures.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Inference code released on GitHub (https://github.com/deepseek-ai/DeepSeek-V3 and https://github.com/deepseek-ai/DeepSeek-R1). Model weights released on HuggingFace. Section I provides download and usage instructions."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section I states 'We also release SFT and RL data to the public.' The paper uses many standard public benchmarks (MMLU, AIME, Codeforces, etc.) and releases model weights on HuggingFace."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section I mentions 'pip install -r requirements.txt' but does not provide the actual environment specification or library versions in the paper itself. No Dockerfile or detailed environment setup section."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section I provides step-by-step commands for downloading weights, cloning the repo, installing dependencies, converting model weights, and running inference. Hardware requirements (16 H800 GPUs) are specified."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables report point estimates only (e.g., '79.8% AIME 2024'). No confidence intervals or error bars are provided despite multiple samples being generated. Bold numbers indicate t-test significance but no CIs."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3, 8, and 15 state 'Numbers in bold denote the performance is statistically significant (t-test with p < 0.01).' Statistical significance is used for comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results consistently report absolute scores with baselines for context (e.g., AIME from 15.6% initial to 77.9% after RL, Codeforces 96.3 percentile vs 58.7 for V3). Percentage improvements and rating differences are provided throughout Tables 3, 8, 12."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for evaluation sample sizes. Pass@k uses k=64 for AIME, k=16 for MATH, k=8 for LCB (Section D.1), but no power analysis or justification for why these k values are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as pass@1 averages over k samples but no standard deviations, IQR, or spread measures are provided across runs or seeds. Only point estimates in result tables."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 compares against Claude-3.5-Sonnet, GPT-4o, DeepSeek-V3, OpenAI-o1-mini, and OpenAI-o1-1217. Table 15 compares distilled models against GPT-4o and Claude-3.5-Sonnet."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include OpenAI-o1-1217, GPT-4o-0513, Claude-3.5-Sonnet-1022, and QwQ-32B-Preview — all contemporary models at time of publication (January 2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows results at each pipeline stage (R1-Zero → Dev1 → Dev2 → Dev3 → R1), isolating the contribution of each training phase. Appendix B.6 provides ablation on language consistency reward. Section F.1 compares distillation vs RL."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Extensive multi-metric evaluation: MMLU, MMLU-Pro, GPQA, AIME, MATH-500, LiveCodeBench, Codeforces rating/percentile, SWE-Bench, Aider, AlpacaEval, ArenaHard, IFEval, SimpleQA, FRAMES, and more (Table 8)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section D.2 reports ChatbotArena human evaluation results with Elo rankings. Figure 11 shows style-controlled ranking. The platform uses double-blind pairwise comparisons with millions of user votes."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Multiple held-out test sets used. Section E.2 specifically tests on AIME 2025 (released after training) to verify generalization. LiveCodeBench uses problems from Aug 2024-Jan 2025. Decontamination applied (Section D.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figures 15-17 provide per-category breakdowns of MMLU (by subject), MMLU-Pro (by domain), and math competition problems (by category: algebra, geometry, combinatorics, etc.). Table 14 breaks LiveCodeBench by difficulty."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific failure modes: poor structural output, token inefficiency/overthinking, language mixing, prompt sensitivity, limited software engineering improvement. Section G.2 discusses unsuccessful attempts (PRM, MCTS)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section G.2 reports three failed approaches: Process Reward Models, MCTS, and small-model RL. Figure 6 shows reward hacking (reward increases while Codeforces performance decreases). Appendix B.6 shows language consistency reward slightly degrades code performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims: (1) reasoning via pure RL — supported by R1-Zero results (Section 2, Table 3); (2) emergent self-reflection/verification — supported by Figures 9, Table 2; (3) superior to supervised learning — supported by Table 12 R1 vs V3 comparison; (4) distillation to smaller models — supported by Table 15."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about RL driving reasoning emergence are supported by the ablation structure: R1-Zero (pure RL, no SFT) develops reasoning behaviors tracked across training steps (Figures 1, 8, 9). The multi-stage pipeline (Table 3) isolates each phase's contribution. Ablation in B.6 tests language consistency reward causally."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes broad claims about 'incentivizing reasoning capability in LLMs' but results are specific to DeepSeek-V3-Base architecture. Section G.1 notes smaller models (7B, 16B) failed to show improvements, suggesting the approach may not generalize to all LLMs. The title and abstract are broader than the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section A.1 acknowledges that pre-training data may contain OpenAI-model-generated answers on web pages. Section G.1 discusses base model capacity as a critical factor. The paper acknowledges reward hacking as an alternative to genuine improvement (Section B.5). Section 6 discusses prompt sensitivity affecting results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames benchmark scores (AIME, MATH, Codeforces) as evidence of 'reasoning capability' without discussing the gap between benchmark performance and actual reasoning. No discussion of whether solving math competitions is a valid proxy for general reasoning ability, despite the title claiming general 'reasoning capability.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Model versions specified: DeepSeek-V3-Base (671B/37B active MoE), GPT-4o-0513, Claude-3.5-Sonnet-1022, OpenAI-o1-1217, GPT-4-Turbo-1106 for evaluation. Baselines include snapshot dates. Distilled model base models specified in Table 6."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Extensive prompt text provided: training template (Table 1), evaluation prompts for each benchmark (Tables 18-32), reward model prompt (Appendix B.2), risk review prompt (Listing 8), SFT data generation prompts (Listings 1-4), and cold-start prompts (Listings 5-7)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed hyperparameters throughout: learning rate 3e-6, KL coefficient 0.001, temperature 1/0.7, batch size 512, GRPO clip ratio ε=10, max length 32768/65536, reference model update every 400 steps (Sections 2.1, 3.2). Distillation hyperparameters in Table 6. SFT hyperparameters in B.4.2."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. DeepSeek-R1 is a single model that generates responses directly. The RL training pipeline is infrastructure, not scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section B.3 describes RL data composition (Table 4: 26K math, 17K code, 22K STEM, 15K logic, 66K general). Cold-start data creation (B.3.2) describes filtering pipeline. SFT data statistics in Table 5. Decontamination filtering described in D.1. Code test case generation pipeline in B.3.1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Conclusion, Limitation, and Future Work' contains extensive discussion of specific limitations across multiple paragraphs covering structural output, token efficiency, language mixing, prompt sensitivity, software engineering, reward hacking."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 identifies specific threats: (1) pure RL depends on reliable reward signals which are hard for tasks like writing; (2) language mixing is caused by the base model's Chinese/English training data; (3) few-shot prompting degrades R1 performance; (4) limited software engineering RL data. Section G.2 details specific failed approaches."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states what R1 cannot do: structural output, tool use, token efficiency for simple problems, non-Chinese/English languages. Section G.1 states smaller models failed, bounding the approach to large-scale models. Acknowledges reward hacking as unsolved for complex tasks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Training data details are described but the actual training data is not fully released. Section I mentions SFT and RL data will be released but at time of paper this appears incomplete ('released at xxx' placeholder). Pre-training data is proprietary web crawl data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section B.3 describes RL data collection across all categories with counts, sources, and characteristics. Cold-start data pipeline (B.3.2) describes DeepSeek-R1-Zero generation → filtering → DeepSeek-V3 refinement → human verification. Code test case generation in B.3.1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. ChatbotArena is an external platform. The paper uses standard benchmarks and model-generated data."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 illustrates the full multi-stage pipeline. Section B.3.2 documents cold-start data: prompts → R1-Zero generation (temp=1.0) → correctness filtering (sympy) → format filtering (repetition, language mixing) → V3 refinement → human verification. Table 5 provides final data statistics."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section mentioning funding sources. DeepSeek is a commercial AI company but no explicit funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed under 'DeepSeek-AI' with the email research@deepseek.com. The affiliation is clear — this is a company paper evaluating its own product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "DeepSeek-AI is a commercial AI company evaluating its own model. The funder (DeepSeek) has a direct financial interest in positive results. No independent funding or external validation mentioned."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement. DeepSeek is a for-profit company whose commercial product is the subject of the paper. No disclosure of patents, equity, or other financial interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section D.1 states 'DeepSeek-V3 base has a knowledge cutoff date of July 2024, predating evaluation benchmarks like CNMO 2024.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section D.1 describes comprehensive decontamination: 'filtered out any text segments (including web pages and GitHub files) that contained matching 10-gram sequences from evaluation questions or reference solutions.' Math domain alone identified ~6 million potential texts for removal."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section D.1 addresses contamination: n-gram filtering for pre-training and post-training data, temporal separation (math SFT from pre-2023 competitions only), and honest acknowledgment that 'n-gram based decontamination method cannot prevent the paraphrase of testset.' Section E.2 tests on AIME 2025 as fresh validation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study. ChatbotArena is an external crowdsourced platform, not a study run by the authors."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study conducted by the authors."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study conducted by the authors."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study conducted by the authors."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study conducted by the authors."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study conducted by the authors."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study conducted by the authors."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example token consumption reported despite the model generating thousands of thinking tokens per problem. Figure 18 shows token counts but not wall-clock time or dollar cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 reports total training costs: 101K H800 GPU hours for R1-Zero, 5K for SFT data creation, 41K for R1, totaling 147K GPU hours ($294K at $2/GPU-hour). Hardware specified as 64×8 H800 GPUs, with R1-Zero training taking ~198 hours."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results reported across multiple random seeds. Training curves (Figure 1) show single runs. Evaluation uses pass@k sampling but does not report sensitivity to random seeds in training."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section D.1 specifies: k=64 for AIME and GPQA, k=16 for MATH and CodeForces, k=8 for LCB. Pass@1 formula is provided. Reward model trained for a single epoch (B.2). Training is 10,400 steps / 1.6 epochs for R1-Zero."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. Section B.4 lists final hyperparameters but does not describe how they were selected or how many configurations were tried. The paper mentions smaller model experiments (7B, 16B, 32B, 230B) but not systematic search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents final configurations without explaining how they were selected. For example, GRPO clip ratio ε=10 is described as crucial but the selection process is not documented."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many pairwise comparisons across ~20 benchmarks and 6+ models with t-tests (p<0.01) but no multiple comparison correction (Bonferroni, Holm, etc.) is mentioned."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "DeepSeek employees evaluate DeepSeek models against competitors. No acknowledgment of self-comparison bias. Baselines from other companies are evaluated using the authors' framework. No independent evaluation mentioned beyond ChatbotArena."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison at matched compute budgets. DeepSeek-R1 uses long chains of thought (8K-18K tokens per problem, Figure 18) vs baselines that use much less compute. The 671B model is compared against models of various sizes without compute normalization."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmarks like AIME, MATH, or Codeforces actually measure 'reasoning capability' as claimed. The paper equates benchmark performance with reasoning without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding involved. The model generates responses directly. SWE-Bench uses the agentless framework consistently (Section D.1)."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section D.1 states training cutoff is July 2024 and notes benchmarks like CNMO 2024 postdate it. Math SFT prompts sourced exclusively from pre-2023 competitions. Section E.2 tests on AIME 2025 as temporal validation."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation prompts or framework setups leak information. For example, AlpacaEval and ArenaHard use GPT-4 as judge, and the evaluation prompt format could advantage certain response styles."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark problems share structural similarities with training data. Acknowledged that web pages may contain 'OpenAI-model-generated answers' (Section A.1) but not analyzed for benchmark-specific overlap beyond n-gram filtering."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section D.1 describes concrete decontamination: 10-gram sequence matching against evaluation questions and reference solutions, removing ~6 million pre-training texts in math alone. Post-training data undergoes same n-gram protocol. Temporal split (pre-2023 math prompts only)."
       }
     }
   },
   "claims": [
     {
       "claim": "Pure RL without SFT can incentivize reasoning capabilities in LLMs, with DeepSeek-R1-Zero achieving 79.8% pass@1 on AIME 2024",
       "evidence": "Figure 1(a) shows AIME accuracy increasing from 15.6% to 77.9% during RL training. Table 8 shows final R1 at 79.8%. Self-reflection and verification behaviors emerge naturally (Table 2, Figures 9).",
       "supported": "strong"
     },
     {
       "claim": "DeepSeek-R1 matches or surpasses OpenAI-o1-1217 on math and coding benchmarks",
       "evidence": "Table 8: R1 achieves 79.8% vs o1's 79.2% on AIME, 97.3% vs 96.4% on MATH-500, 65.9% vs 63.4% on LiveCodeBench. Codeforces: 96.3 percentile vs 96.6. Some benchmarks show o1 ahead (SimpleQA, Aider).",
       "supported": "strong"
     },
     {
       "claim": "Distilled small models (1.5B-70B) outperform GPT-4o and Claude-3.5-Sonnet on math benchmarks",
       "evidence": "Table 15: Even 1.5B distilled model achieves 28.9% on AIME vs GPT-4o's 9.3% and 83.9% on MATH vs 74.6%. Larger distilled models show progressively better results.",
       "supported": "strong"
     },
     {
       "claim": "Distillation outperforms RL alone for smaller models",
       "evidence": "Table 16: DeepSeek-R1-Distill-Qwen-32B (72.6% AIME) significantly outperforms Qwen2.5-32B-Zero (47.0% AIME) which used RL directly on the 32B base.",
       "supported": "strong"
     },
     {
       "claim": "Emergent reasoning behaviors (self-reflection, verification) develop autonomously during RL training",
       "evidence": "Figure 9 shows 5-7x increase in reflective word frequency. Table 2 shows 'aha moment' with 'wait' usage. Figure 1(b) shows increasing response length. However, these are correlational — hard to distinguish genuine reasoning from pattern mimicry.",
       "supported": "moderate"
     },
     {
       "claim": "The total training cost for DeepSeek-R1 is approximately $294K in GPU hours",
       "evidence": "Table 7 provides detailed breakdown: 101K H800 GPU hours for R1-Zero, 41K for R1, 5K for SFT data at $2/GPU-hour.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "DeepSeek-AI employees evaluate DeepSeek models. No independent evaluation or external audit. While ChatbotArena provides some independent signal, the main benchmark evaluations are self-conducted."
     },
     {
       "flag": "No variance or uncertainty reporting",
       "detail": "Despite using pass@k evaluation with multiple samples, no standard deviations, confidence intervals, or result stability measures are reported across any benchmark. The reader cannot assess how reliable any single number is."
     },
     {
       "flag": "Selective OpenAI-o1 comparison",
       "detail": "The paper acknowledges 'accessing the OpenAI-o1-1217 API is challenging in mainland China' and reports o1 performance 'based on official reports.' This means some comparisons use self-reported numbers from the competitor rather than controlled evaluation."
     },
     {
       "flag": "Missing multiple comparison correction",
       "detail": "T-tests with p<0.01 are run across 20+ benchmarks and 6+ model comparisons without any family-wise error rate correction. With this many comparisons, some significant results could be spurious."
     },
     {
       "flag": "Benchmark-reasoning gap not discussed",
       "detail": "The paper equates benchmark performance on AIME/MATH/Codeforces with 'reasoning capability' without discussing whether these benchmarks are valid proxies for general reasoning. The claimed 'reasoning' could be sophisticated pattern matching on competition-style problems."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": [
         "M. Chen",
         "J. Tworek"
       ],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and pass@k evaluation methodology used throughout this paper."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": [
         "L. Ouyang",
         "J. Wu"
       ],
       "year": 2022,
       "relevance": "Foundational RLHF work establishing the SFT→RL pipeline that DeepSeek-R1 modifies by skipping SFT."
     },
     {
       "title": "Scaling LLM test-time compute optimally can be more effective than scaling model parameters",
       "authors": [
         "C. Snell",
         "J. Lee"
       ],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Test-time compute scaling framework directly relevant to DeepSeek-R1's adaptive token generation strategy."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": [
         "J. Wei",
         "X. Wang"
       ],
       "year": 2022,
       "relevance": "Chain-of-thought prompting foundation that DeepSeek-R1 aims to surpass via RL-emergent reasoning."
     },
     {
       "title": "DeepSeek-V3 technical report",
       "authors": [
         "DeepSeek-AI"
       ],
       "year": 2024,
       "arxiv_id": "2412.19437",
       "relevance": "Base model for DeepSeek-R1; provides the 671B MoE architecture and pre-training details."
     },
     {
       "title": "Agentless: Demystifying LLM-based software engineering agents",
       "authors": [
         "C. S. Xia",
         "Y. Deng"
       ],
       "year": 2024,
       "relevance": "Framework used for SWE-Bench Verified evaluation of DeepSeek-R1."
     },
     {
       "title": "Let's verify step by step",
       "authors": [
         "H. Lightman",
         "V. Kosaraju"
       ],
       "year": 2024,
       "relevance": "Process reward model approach that DeepSeek-R1 considered but rejected in favor of outcome-based RL."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": [
         "X. Wang",
         "J. Wei"
       ],
       "year": 2023,
       "relevance": "Self-consistency decoding used to boost DeepSeek-R1-Zero's AIME score from 77.9% to 86.7%."
     },
     {
       "title": "DeepSeekMath: Pushing the limits of mathematical reasoning in open language models",
       "authors": [
         "Z. Shao",
         "P. Wang"
       ],
       "year": 2024,
       "arxiv_id": "2402.03300",
       "relevance": "Introduces GRPO algorithm adopted for DeepSeek-R1 training."
     },
     {
       "title": "Distillation scaling laws",
       "authors": [
         "D. Busbridge"
       ],
       "year": 2025,
       "arxiv_id": "2502.08606",
       "relevance": "Scaling laws for knowledge distillation, supporting DeepSeek-R1's distillation approach."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": [
         "N. Jain",
         "K. Han"
       ],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Key code generation benchmark used for evaluation with temporal contamination prevention."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI feedback",
       "authors": [
         "Y. Bai",
         "A. Jones"
       ],
       "year": 2022,
       "relevance": "RLHF safety alignment methodology that DeepSeek-R1's safety training builds upon."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Open-weight models (1.5B-70B) released on HuggingFace that practitioners can immediately download and use for reasoning tasks."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Pure RL without SFT producing emergent reasoning behaviors and matching OpenAI-o1 challenges the assumption that supervised fine-tuning on human demonstrations is necessary."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Paper acknowledges jailbreak vulnerabilities and enhanced capability for dangerous content but treats safety as secondary to the technical contribution."
     },
     "drama_conflict": {
       "score": 3,
       "justification": "A Chinese lab openly challenges OpenAI's flagship reasoning model, claims comparable performance at a fraction of the cost ($294K), and releases everything under MIT license."
     },
     "demo_ability": {
       "score": 3,
       "justification": "All model weights from 1.5B to 671B are publicly available on HuggingFace with inference code and instructions, and a hosted API exists."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "DeepSeek-R1 became a global news story, directly competing with OpenAI's o1, and is one of the most discussed AI releases of 2025."
     }
   }
 }