ai-research-survey

scan.json (33366B)

---

 {
   "paper": {
     "title": "Is DPO Superior to PPO for LLM Alignment? A Comprehensive Study",
     "authors": [
       "Shusheng Xu",
       "Wei Fu",
       "Jiaxuan Gao",
       "Wenjie Ye",
       "Weilin Liu",
       "Zhiyu Mei",
       "Guangju Wang",
       "Chao Yu",
       "Yi Wu"
     ],
     "year": 2024,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2404.10719",
     "doi": "10.48550/arXiv.2404.10719"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "Through theoretical analysis and empirical validation, the paper shows that DPO suffers from distribution shift between base model outputs and preference data, causing it to favor out-of-distribution responses. PPO with three key techniques — advantage normalization, large batch size, and exponential moving average reference model updates — consistently outperforms DPO across dialogue (HH-RLHF, SafeRLHF) and code generation (APPS, CodeContest) tasks. On CodeContest, PPO with CodeLlama-34B achieves 22.4% pass@10@1k, surpassing AlphaCode-41B's 16.4%. Iterative DPO partially mitigates distribution shift but still underperforms PPO, especially on challenging code tasks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our code is publicly available at https://github.com/openpsi-project/ReaLHF' and provides a working URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used (HH-RLHF, SafeRLHF, APPS, CodeContest) are publicly available standard benchmarks. The paper references public dataset URLs and official codebases."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions DeepSpeed-Chat as the base implementation but does not provide a requirements.txt, Dockerfile, or detailed environment specification listing library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released, the paper does not include step-by-step reproduction instructions or refer to a README with specific commands to replicate the main experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All result tables (Tables 2-8, 9-14) report only point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims PPO 'outperforms' and 'surpasses' DPO based solely on comparing raw numbers across all experiments. No statistical significance tests (t-tests, bootstrap, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute improvements with baseline context throughout — e.g., '10@1k improvement from 16.4% to 22.4%' (Section 1), safety rate increases from 55.4% to 99.5% (Table 2), and pass@5 improvements from 38.6% to 44.4% (Table 7). These provide magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for sample sizes. The human evaluation uses only 100 queries evaluated by 4 people (Appendix B/C.4) with no power analysis or justification for why this sample is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures across experimental runs are reported anywhere in the paper. All results appear to be single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against SFT (base), DPO, DPO-Iter, RRHF, PRO (Table 4), and prior systems like AlphaCode (Table 8) and CodeRL (Table 7)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "DPO (2023), RRHF (2023), PRO (2023) are contemporary methods for a 2024 paper. AlphaCode (2022) is the state-of-the-art on CodeContest. The baselines represent the state of the art for the tasks tested."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 presents a systematic ablation of PPO components: baseline PPO → + advantage normalization → + large batch size → + reference EMA. Figure 2 further ablates batch size across multiple values."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "HH-RLHF uses OpenAssistant reward, win rates vs chosen, and win rates vs SFT (Table 4). SafeRLHF uses helpfulness reward, harmfulness score, and safety rate (Table 6). Code tasks use pass@k at multiple k values (Tables 7-8). GPT-4 and human evaluation provide additional metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Table 14 (Appendix C.4) reports human evaluation on HH-RLHF: 100 queries evaluated by 4 different people each, comparing PPO vs DPO and PPO vs DPO-Iter."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on test sets: 'Results on the HH-RLHF test set' (Table 4), 'Results on Apps test set' (Table 7), CodeContest reports validation and test set separately (Table 8). PPO checkpoints are selected on validation sets (Appendix A.2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "APPS results are broken down by difficulty level (Introductory, Interview, Competition) in Table 7. SafeRLHF separates helpfulness and harmfulness scores (Table 6). HH-RLHF shows win/tie/lose breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "DPO failures are discussed extensively: DPO achieves 0% pass rate on CodeContest (Table 8), DPO outputs 'many meaningless code snippets' (Section 6). Section 4 analyzes why DPO fails through distribution shift. Baseline PPO with small batch size degrades below SFT (Table 3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: baseline PPO with small batch size shows 'significant performance degradation on the APPS dataset' (Section 5). DPO-Iter fails to improve SFT on code tasks across all model sizes (Table 7). Filtering dual-safe data hurts helpfulness for DPO (Table 2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims PPO 'surpass[es] other alignment methods in all cases' and achieves 'state-of-the-art results in challenging code competitions.' Tables 4-8 confirm PPO outperforms across all tested settings, and Table 8 shows CodeLlama-34B PPO (22.4%) surpasses AlphaCode-41B (16.4%) on CodeContest."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about PPO components are supported by controlled ablation (Table 3) where components are added one at a time. The theoretical analysis (Theorem 4.1) provides formal justification for DPO's distribution shift issue. The ablation design with single-variable manipulation is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'A Comprehensive Study' but only Llama-family models (Llama 1 7B, Llama 2 7B, CodeLlama 7B/13B/34B) are tested. The conclusion states 'PPO demonstrates robust effectiveness across diverse tasks' without bounding to Llama models. No other model families (e.g., Mistral, Pythia, GPT-class) are evaluated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for PPO's superiority. For example, it does not consider whether PPO's advantage comes from higher compute budget, whether the ReaLHF framework is better optimized for PPO, or whether different hyperparameter search effort between PPO and DPO explains the gap."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "For code tasks, pass@k directly measures correctness against test cases — no proxy gap. For dialogue, the paper uses multiple evaluation approaches (reward model, GPT-4, human evaluation) and explicitly notes the OpenAssistant reward model 'is not used during training' (Section 6), distinguishing training proxy from evaluation metric."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific open-source model identifiers are used throughout: 'Llama2-7B', 'Llama 1 7B', 'CodeLlama-7B', 'CodeLlama-13B', 'CodeLlama-34B' (Tables 3, 6, 7). These are precisely defined open-source model releases with deterministic weights."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The GPT-4 evaluation prompt is fully reproduced in Appendix B with exact text. Training prompts come from public datasets (HH-RLHF, SafeRLHF, APPS, CodeContest) which are fully referenced."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 reports comprehensive PPO hyperparameters: learning rates (1e-5 actor, 5e-6 critic), batch size (512), temperature (1.0), top-k (200), GAE λ=1, γ=1, β=0.1, reward clip=20. Appendix A.1 reports DPO hyperparameters: β=0.1, learning rate 1e-6."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates standard RLHF fine-tuning pipelines (PPO and DPO) applied directly to language models."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes how SafeRLHF preference data is constructed (safety-priority ordering with binary safety labels). Section 6 describes how APPS/CodeContest data is processed for PPO (test-case rewards) and DPO-Iter (sampling 5 codes, labeling with test cases, using dataset solutions as yw when no correct sample exists)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Conclusion) contains a substantive limitations paragraph: 'There are also limitations in our work. The reward model is significant in the training processes of both PPO and DPO-Iter. However, in this paper, we have not delved into the discussion of how to effectively train a robust reward model.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: the reward model quality was not studied, and ground-truth rewards were used for code tasks ('For the code competition task, we utilize the ground-truth reward for PPO training and the labeling of DPO-Iter'). These are concrete, study-specific limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound conclusions to Llama-family models, does not state which other model architectures or training regimes are excluded, and does not address whether results generalize beyond the tested task domains."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All datasets (HH-RLHF, SafeRLHF, APPS, CodeContest) are publicly available standard benchmarks. The paper cites official repositories and download sources. Code is released at GitHub."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes the SafeRLHF dataset structure and how preferences are derived. Section 6 describes each dataset's format and how training signals are generated. The synthetic scenario in Section 4.2 describes its construction explicitly."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation (Table 14) involves 4 evaluators rating 100 queries, but the paper does not describe who these evaluators are, how they were recruited, or whether their backgrounds could introduce bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw datasets to training signals is documented: SafeRLHF preference construction (Section 4.3), APPS/CodeContest reward generation from test cases (Section 6), DPO-Iter data collection process (sampling 5 codes, labeling, using dataset solutions for missing correct samples), and checkpoint selection via validation set (Appendix A.2)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. Authors are from Tsinghua University, OpenPsi Inc., and Shanghai Qi Zhi Institute, but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Tsinghua University, OpenPsi Inc., and Shanghai Qi Zhi Institute. The code is released under the openpsi-project GitHub organization, linking authors to the product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence cannot be assessed. Some authors are affiliated with OpenPsi Inc., which develops the ReaLHF framework that implements the PPO approach advocated by the paper, creating a potential commercial interest in PPO's favorable results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. OpenPsi Inc. affiliation and the ReaLHF product could represent a financial interest related to the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for Llama 2 or CodeLlama, despite these pre-trained models being evaluated on benchmarks (APPS from 2021, HH-RLHF from 2022) that pre-date the models' training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether APPS, CodeContest, or HH-RLHF test examples appeared in Llama 2 or CodeLlama's pre-training data, despite these benchmarks being publicly available before the models were trained."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "APPS (2021), HH-RLHF (2022), and CodeContest (2022) were all published before Llama 2's training cutoff. The paper does not address whether these benchmarks' test cases were seen during pre-training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the human evaluation study. The paper does not reference OSF, AsPredicted, or any pre-registration platform."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the human evaluation component involving 4 evaluators rating model outputs."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation (Table 14) states '4 different persons' evaluated each pair, but provides no information about evaluators' backgrounds, expertise, language proficiency, or demographics."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No criteria are stated for who was eligible to serve as a human evaluator. The paper simply states 'each reference pairs are evaluated by 4 different persons' without selection criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The human evaluators are not assigned to experimental conditions — all evaluators perform the same pairwise comparison task. Randomization of participants to conditions is not applicable."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "For GPT-4 evaluation, position swapping is described (Appendix B). For the human evaluation (Table 14), blinding is not explicitly described — evaluators presumably saw anonymized 'Response A/B' labels, but this is not stated."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition or dropout information is reported for the human evaluation. It is unclear whether any evaluator judgments were excluded or how disagreements beyond GPT-4 agreement rates were handled."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, generation latency, or per-example costs are reported. The paper samples 1000 codes per problem for pass@1k evaluation without reporting the associated compute cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total training time, or hardware specifications are reported, despite training models up to 34B parameters with PPO for 16 epochs and needing a separate reward model."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as single point estimates. No seed sensitivity analysis or results across multiple random seeds are presented, despite Henderson et al. (2018) showing RL results can vary significantly across seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of training runs for each experiment is not stated. It is unclear whether results come from single runs or averages. The pass@k metric implies multiple samples per problem, but the number of training runs is unstated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "For DPO, 'We sweep the batch size and report the best performance' (Appendix A.1) but the number of configurations tried is not stated. For PPO, hyperparameters appear tuned but no search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 states 'The checkpoints with the highest reward/pass@k on the validation sets are selected,' which is proper validation-based selection. The PPO ablation (Table 3) systematically shows which components contribute."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, making correction for multiple comparisons moot. The absence of tests is captured by significance_tests."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines (DPO, RRHF, PRO) using their own ReaLHF framework. They do not acknowledge the bias of evaluating their own implementation of PPO against their own implementations of baselines, per Lucic et al. (2018)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "PPO requires training a separate reward model plus multi-epoch RL training, which is significantly more compute than DPO's single-pass optimization. This compute difference is never quantified or discussed, yet the paper claims PPO is 'superior.'"
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses APPS, CodeContest, HH-RLHF, and SafeRLHF without discussing whether these benchmarks validly measure the claimed capabilities (e.g., whether pass@k on competitive programming reflects general code generation ability)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved — the paper evaluates direct model fine-tuning and generation without agentic scaffolds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether Llama 2 or CodeLlama pre-training data includes solutions from APPS (2021) or CodeContest (2022) problems, despite temporal overlap being likely."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "For DPO-Iter on APPS, when the base model cannot sample any correct answer, 'we use the correct solutions from the dataset as yw' (Section 6). The paper does not discuss whether this introduces information leakage from the dataset's solution distribution."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and test problems in APPS or CodeContest share structural similarities, come from the same programming platforms, or have near-duplicate problems."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods (canary strings, membership inference, decontamination) are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "DPO may find biased solutions that exploit out-of-distribution responses, and ΠPPO is a proper subset of ΠDPO (the set of DPO solutions is strictly larger than PPO solutions).",
       "evidence": "Theorem 4.1 with formal proof (Section 4.1), counter-example in Table 1 showing DPO can assign 0.9 probability to an action with zero reference probability, and empirical validation in synthetic scenario (Section 4.2, Figure 1).",
       "supported": "strong"
     },
     {
       "claim": "DPO is sensitive to distribution shift between the base model outputs and the preference dataset, and mitigating this shift improves DPO performance.",
       "evidence": "Table 2 shows DPO safety rate improves from 55.4% to 71.8% when using SFT(Safe) as reference model instead of SFT(Alpaca). Tables 9-10 (Appendix C.1) show DPO performance varies dramatically with reference model choice (0.24% vs 12.8% on APPS).",
       "supported": "strong"
     },
     {
       "claim": "PPO consistently outperforms DPO across all tested tasks including dialogue and code generation.",
       "evidence": "Table 4: PPO achieves 0.718 reward vs DPO's 0.611 on HH-RLHF. Table 6: PPO achieves 99.5% safety rate with +1.69 helpfulness vs DPO's 95.8% with -2.86 on SafeRLHF. Table 7: PPO outperforms DPO-Iter at all model sizes on APPS. Table 8: PPO achieves 22.4% vs DPO-Iter's 3.2% on CodeContest.",
       "supported": "moderate"
     },
     {
       "claim": "Three key factors improve PPO for RLHF: advantage normalization, large batch size, and exponential moving average update for the reference model.",
       "evidence": "Table 3 ablation: baseline PPO → +Adv.Norm → +Large.Batch → +Ref.EMA shows progressive improvement on HH-RLHF (0.706→0.718), APPS Intro (18.0%→44.4%), and CodeContest (4.3%→6.8% pass@10). Figure 2 shows batch size impact across difficulty levels.",
       "supported": "moderate"
     },
     {
       "claim": "PPO with CodeLlama-34B achieves state-of-the-art results on CodeContest, outperforming AlphaCode-41B.",
       "evidence": "Table 8: PPO CodeLlama-34B achieves 22.4% pass@10@1k on the test set vs AlphaCode-41B's 16.4% (with clustering). However, PPO uses ground-truth test-case rewards while AlphaCode does not have this advantage.",
       "supported": "moderate"
     },
     {
       "claim": "Iterative DPO can partially close the gap with PPO on dialogue tasks but fails on challenging code generation tasks.",
       "evidence": "Table 2: DPO-Iter achieves 99.9% safety rate (comparable to PPO's 99.5%) but lower helpfulness (-2.96 vs +1.69). Table 7: DPO-Iter fails to improve over SFT on APPS across all model sizes. Table 8: DPO-Iter achieves only 3.2% vs PPO's 22.4% on CodeContest.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance across runs",
       "detail": "All experimental results across all tables are single point estimates without standard deviations, confidence intervals, or any measure of variance. For a paper making strong comparative claims ('PPO surpasses DPO in all cases'), the absence of uncertainty quantification is a significant concern, especially for RL methods known to have high variance across seeds."
     },
     {
       "flag": "Unequal comparison conditions",
       "detail": "For code generation tasks, PPO directly optimizes ground-truth test-case rewards (binary pass/fail), while DPO-Iter must work through preference pairs constructed from sampled solutions. When the base model fails to sample correct solutions, dataset solutions are used as yw for DPO-Iter. This creates a structural advantage for PPO that is not acknowledged or controlled for."
     },
     {
       "flag": "Missing compute budget comparison",
       "detail": "PPO requires training a separate reward model plus multi-epoch RL training with actor and critic networks, while DPO requires only single-pass preference optimization. The paper claims PPO is 'superior' without ever reporting or comparing the compute costs, making it impossible to assess whether PPO's gains justify its additional resource requirements."
     },
     {
       "flag": "Self-comparison bias",
       "detail": "All methods (PPO, DPO, RRHF, PRO) are implemented by the authors in their own ReaLHF framework. The PPO implementation may be more carefully optimized than the baseline implementations, as the authors developed ReaLHF specifically for PPO-based RLHF. This bias is not acknowledged."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims of PPO outperforming DPO rest entirely on comparing raw numbers without any statistical testing. Given the known high variance of RL algorithms (Henderson et al., 2018) and the absence of multi-seed experiments, the observed differences could potentially be within noise."
     }
   ],
   "cited_papers": [
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell", "Stefano Ermon", "Christopher D. Manning", "Chelsea Finn"],
       "year": 2023,
       "arxiv_id": "2305.18290",
       "relevance": "The core DPO method analyzed and compared against PPO in this paper; foundational to the reward-free RLHF approach."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "The InstructGPT paper that established the PPO-based RLHF pipeline for aligning LLMs with human preferences."
     },
     {
       "title": "Fine-tuning language models from human preferences",
       "authors": ["Daniel M. Ziegler", "Nisan Stiennon", "Jeffrey Wu"],
       "year": 2019,
       "arxiv_id": "1909.08593",
       "relevance": "Early work on RLHF for language models that pioneered the reward model + PPO approach."
     },
     {
       "title": "Scaling laws for reward model overoptimization",
       "authors": ["Leo Gao", "John Schulman", "Jacob Hilton"],
       "year": 2023,
       "relevance": "Studies reward model overoptimization in RLHF, directly relevant to understanding PPO failure modes."
     },
     {
       "title": "Open problems and fundamental limitations of reinforcement learning from human feedback",
       "authors": ["Stephen Casper", "Xander Davies", "Claudia Shi"],
       "year": 2023,
       "arxiv_id": "2307.15217",
       "relevance": "Comprehensive survey of RLHF limitations including reward model quality and optimization challenges."
     },
     {
       "title": "Safe rlhf: Safe reinforcement learning from human feedback",
       "authors": ["Josef Dai", "Xuehai Pan", "Ruiyang Sun"],
       "year": 2023,
       "relevance": "Source of the SafeRLHF dataset and safe alignment methodology used as a primary testbed in this paper."
     },
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Yuntao Bai", "Andy Jones", "Kamal Ndousse"],
       "year": 2022,
       "relevance": "Source of the HH-RLHF dataset and foundational work on helpful and harmless LLM alignment."
     },
     {
       "title": "Competition-level code generation with alphacode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "relevance": "State-of-the-art code generation system on CodeContest that PPO claims to surpass in this paper."
     },
     {
       "title": "Secrets of RLHF in large language models part I: PPO",
       "authors": ["Rui Zheng", "Shihan Dou", "Songyang Gao"],
       "year": 2023,
       "arxiv_id": "2307.04964",
       "relevance": "Closely related work studying PPO implementation details for LLM RLHF training."
     },
     {
       "title": "Learning to summarize with human feedback",
       "authors": ["Nisan Stiennon", "Long Ouyang", "Jeff Wu"],
       "year": 2020,
       "relevance": "Early demonstration of RLHF applied to text summarization, establishing the reward model + PPO paradigm."
     },
     {
       "title": "Self-rewarding language models",
       "authors": ["Weizhe Yuan", "Richard Yuanzhe Pang", "Kyunghyun Cho"],
       "year": 2024,
       "arxiv_id": "2401.10020",
       "relevance": "Related iterative self-improvement approach for LLM alignment, relevant to the iterative DPO comparison."
     },
     {
       "title": "DeepSpeed-Chat: Easy, fast and affordable RLHF training of chatgpt-like models at all scales",
       "authors": ["Zhewei Yao", "Reza Yazdani Aminabadi", "Olatunji Ruwase"],
       "year": 2023,
       "arxiv_id": "2308.01320",
       "relevance": "The RLHF training framework that serves as the base implementation for this paper's PPO experiments."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides actionable PPO training recipes (advantage normalization, large batch size, EMA reference) that practitioners fine-tuning LLMs can directly apply."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the prevailing academic narrative that DPO is simpler and comparably effective to PPO, arguing well-tuned PPO is consistently superior."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the paper focuses on alignment training methodology comparison."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Enters the PPO-vs-DPO community debate but presents findings in measured academic tone without inflammatory claims."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code released on GitHub (ReaLHF), but reproducing results requires significant GPU resources for 34B parameter models."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Published at ICML by Tsinghua University researchers; not a top-tier AI lab but a well-known institution and top venue."
     }
   }
 }