ai-research-survey

scan.json (19704B)

---

 {
   "paper": {
     "title": "DAPO: An Open-Source LLM Reinforcement Learning System at Scale",
     "authors": ["Qiying Yu", "Zheng Zhang", "Ruofei Zhu", "Yufeng Yuan", "Xiaochen Zuo", "Yu Yue", "Weinan Dai", "Tiantian Fan", "Gaohong Liu", "Juncai Liu", "Lingjun Liu", "Xin Liu", "Haibin Lin", "Zhiqi Lin", "Bole Ma", "Guangming Sheng", "Yuxuan Tong", "Chi Zhang", "Mofan Zhang", "Ru Zhang", "Wang Zhang", "Hang Zhu", "Jinhua Zhu", "Hao Zhou", "Jingjing Liu", "Wei-Ying Ma", "Ya-Qin Zhang", "Lin Yan", "Mu Qiao", "Yonghui Wu", "Mingxuan Wang"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.14476"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states it is built on the verl framework (https://github.com/volcengine/verl) and provides a project page (https://dapo-sia.github.io/). The abstract states 'we open-source our training code.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 describes the DAPO-Math-17K dataset and the abstract states they open-source 'a carefully curated and processed dataset.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup with library versions is provided in the paper. Only the verl framework is mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper itself. The paper references a project page but does not include commands or a README-style guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., 50 points on AIME 2024). No confidence intervals or error bars are provided for the main results in Table 1."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims DAPO outperforms DeepSeek-R1-Zero-Qwen-32B (50 vs 47) without any statistical significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows progressive improvements with baseline context: Naive GRPO at 30, each technique adding points up to DAPO at 50, compared to DeepSeek-R1-Zero-Qwen-32B at 47. Absolute improvements are clear from context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for evaluation sample sizes. AIME 2024 has only 30 problems. No power analysis or discussion of whether this is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports avg@32 (averaging over 32 repetitions) but does not report standard deviation, IQR, or any spread measure across these repetitions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares against Naive GRPO and DeepSeek-R1-Zero-Qwen-32B."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "DeepSeek-R1-Zero-Qwen-32B (2025) is a contemporary baseline representing the state of the art at the time."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows a progressive ablation, adding each technique incrementally: Overlong Filtering (+6), Clip-Higher (+2), Soft Overlong Punishment (+3), Token-level Loss (+1), Dynamic Sampling (+8)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports avg@32, pass@32, and cons@32 on AIME 2024 (Figure 1)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates math reasoning via automated rule-based verification. Human evaluation is not relevant for verifiable math correctness."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "AIME 2024 is used as the evaluation benchmark, separate from the DAPO-Math-17K training dataset. Section 4.3 also distinguishes training reward from validation accuracy."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregate AIME 2024 accuracy is reported. No per-problem or per-difficulty breakdown is provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses entropy collapse, overfitting to training set, and Section 3 discusses problems like gibberish/repetitive generation and reward noise from truncated samples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that naive GRPO achieved only 30 points (vs 47 target), discusses entropy collapse, and notes that training reward 'often exhibits little correlation with the accuracy on the validation set, which indicates overfitting.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 50 points on AIME 2024 using Qwen2.5-32B, which is supported by Figure 1 and Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about each technique's contribution are supported by progressive ablation in Table 1, where each component is added incrementally in a controlled manner."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'LLM Reinforcement Learning System at Scale' but results are only on AIME 2024 math problems with a single base model (Qwen2.5-32B). Section 4.1 mentions it 'can be readily transferred to other tasks' without evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the improvements. For example, the progressive ablation order could affect results, and no consideration of whether improvements are due to hyperparameter tuning rather than the proposed techniques."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies Qwen2.5-32B as the base model, which is a specific versioned model name."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This is an RL training paper, not a prompting paper. The method uses rule-based reward signals, not prompt engineering."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports learning rate (1e-6), warm-up steps (20), prompt batch size (512), samples per prompt (16), mini-batch size (512), max length (20480), epsilon_low (0.2), epsilon_high (0.28), temperature (1.0), top_p (0.7)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a direct RL training system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 and Appendix A describe the dataset transformation process: sourcing from web/competitions, transforming answers to integers, using LLM-based reformulation with CoT reasoning. An example is provided."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section is present in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Section 4.1 briefly says 'we focus specifically on mathematical tasks' but does not bound the generalization claims made elsewhere."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the dataset is open-sourced along with training code, enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 describes data sourcing: 'sourced from the web and official competition homepages through a combination of web scraping and manual annotation.'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is math problems from public sources."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 and Appendix A describe the pipeline: source collection, answer format transformation to integers using LLM, resulting in DAPO-Math-17K with 17K prompts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. Authors are from ByteDance Seed and Tsinghua University but no explicit funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: ByteDance Seed, Tsinghua University AIR, The University of Hong Kong, and SIA-Lab."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "ByteDance is a major tech company with commercial interest in LLM capabilities. No discussion of whether the funder has a stake in the results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff of Qwen2.5-32B is not stated. The paper does not discuss when the base model's pretraining data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether AIME 2024 problems could appear in Qwen2.5-32B's pretraining data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "AIME 2024 problems are publicly available and could be in the pretraining data of Qwen2.5-32B. This is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency figures are reported despite the method involving large-scale RL training and generation of up to 20,480 tokens per sample."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total training time, or hardware specifications are reported despite this being a large-scale training system."
       }
     }
   },
   "claims": [
     {
       "claim": "DAPO achieves 50 points on AIME 2024 using Qwen2.5-32B base model",
       "evidence": "Figure 1 and Table 1 show AIME 2024 avg@32 score of 50.",
       "supported": "strong"
     },
     {
       "claim": "DAPO outperforms DeepSeek-R1-Zero-Qwen-32B (47 points) using 50% training steps",
       "evidence": "Figure 1 shows DAPO reaching higher accuracy in fewer steps. Table 1 shows 50 vs 47.",
       "supported": "moderate"
     },
     {
       "claim": "Each of the four key techniques contributes to improved performance",
       "evidence": "Table 1 shows progressive ablation: Naive GRPO 30 → +Overlong Filtering 36 → +Clip-Higher 38 → +Soft Overlong Punishment 41 → +Token-level Loss 42 → +Dynamic Sampling 50.",
       "supported": "moderate"
     },
     {
       "claim": "Clip-Higher strategy effectively addresses entropy collapse",
       "evidence": "Figure 2 shows entropy maintained with Clip-Higher vs collapse without it, along with accuracy improvement.",
       "supported": "strong"
     },
     {
       "claim": "Reflective/backtracking reasoning behaviors emerge during RL training",
       "evidence": "Section 4.4 and Table 2 show qualitative examples of reflection emerging ('wait a moment, let's rethink').",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DAPO proposes four techniques (Clip-Higher, Dynamic Sampling, Token-Level Policy Gradient Loss, Overlong Reward Shaping) that improve GRPO-based RL training for LLM reasoning, achieving 50 points on AIME 2024 with Qwen2.5-32B compared to DeepSeek-R1-Zero's 47 points. The progressive ablation shows each technique contributes, with Dynamic Sampling providing the largest single improvement (+8 points). The work is notable for open-sourcing training code, algorithm details, and a curated 17K math dataset, aiming to improve reproducibility in the LLM RL space.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "Despite using avg@32 (32 repetitions), no standard deviation or confidence interval is reported. A 3-point improvement (47→50) on 30 AIME problems could be within noise."
     },
     {
       "flag": "Single benchmark evaluation",
       "detail": "All results are on AIME 2024 only (30 problems). No evaluation on other math benchmarks (MATH, GSM8K, etc.) or other domains despite claims of general RL system."
     },
     {
       "flag": "Progressive ablation order dependency",
       "detail": "The ablation in Table 1 adds techniques in a fixed order. Individual technique contributions may differ if added in different orders or combinations, but no such analysis is provided."
     },
     {
       "flag": "No compute cost disclosure",
       "detail": "A paper about 'large-scale RL' does not report GPU hours, hardware, or training cost, making practical reproducibility assessment impossible."
     },
     {
       "flag": "Benchmark contamination risk",
       "detail": "AIME 2024 problems are publicly available and could be in Qwen2.5-32B's pretraining data. No contamination analysis is performed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no discussion of limitations, threats to validity, or scope boundaries."
     }
   ],
   "cited_papers": [
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Primary baseline and motivation; key work on RL for LLM reasoning whose results DAPO aims to reproduce and surpass."
     },
     {
       "title": "DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models",
       "authors": ["Zhihong Shao", "Peiyi Wang", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2402.03300",
       "relevance": "Introduces GRPO algorithm that DAPO builds upon and modifies."
     },
     {
       "title": "Proximal Policy Optimization Algorithms",
       "authors": ["John Schulman", "Filip Wolski", "Prafulla Dhariwal"],
       "year": 2017,
       "arxiv_id": "1707.06347",
       "relevance": "Foundation RL algorithm that DAPO's clipping mechanism extends."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF work that DAPO's approach relates to for LLM alignment."
     },
     {
       "title": "HybridFlow: A Flexible and Efficient RLHF Framework",
       "authors": ["Guangming Sheng", "Chi Zhang"],
       "year": 2024,
       "arxiv_id": "2409.19256",
       "relevance": "The verl framework that DAPO's implementation is built upon."
     },
     {
       "title": "RLEF: Grounding Code LLMs in Execution Feedback with Reinforcement Learning",
       "authors": ["Jonas Gehring", "Kunhao Zheng"],
       "year": 2025,
       "relevance": "Related work applying RL with execution feedback for code LLMs."
     },
     {
       "title": "Kimi k1.5: Scaling Reinforcement Learning with LLMs",
       "authors": ["Kimi Team"],
       "year": 2025,
       "arxiv_id": "2501.12599",
       "relevance": "Contemporary work on scaling RL for LLM reasoning, cited as withholding key training details."
     },
     {
       "title": "VinePPO: Unlocking RL Potential for LLM Reasoning through Refined Credit Assignment",
       "authors": ["Amirhossein Kazemnejad", "Milad Aghajohari"],
       "year": 2024,
       "arxiv_id": "2410.01679",
       "relevance": "Related approach to improving credit assignment in RL for LLM reasoning."
     },
     {
       "title": "An Empirical Study on Eliciting and Improving R1-like Reasoning Models",
       "authors": ["Zhipeng Chen", "Yingqian Min"],
       "year": 2025,
       "arxiv_id": "2503.04548",
       "relevance": "Community effort to reproduce DeepSeek R1 results, documenting similar challenges to those DAPO addresses."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn", "Federico Cassano"],
       "year": 2023,
       "relevance": "Related work on using reinforcement learning signals for LLM self-improvement in code generation."
     }
   ]
 }