ai-research-survey

scan.json (30498B)

---

 {
   "paper": {
     "title": "Improving LLM General Preference Alignment via Optimistic Online Mirror Descent",
     "authors": [
       "Yuheng Zhang",
       "Dian Yu",
       "Tao Ge",
       "Linfeng Song",
       "Zhichen Zeng",
       "Haitao Mi",
       "Nan Jiang",
       "Dong Yu"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2502.16852",
     "doi": "10.48550/arXiv.2502.16852"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "ONPO integrates optimistic online mirror descent into self-play RLHF, achieving an O(T⁻¹) duality gap bound improving upon the previous O(T⁻¹/²) result. On AlpacaEval 2.0, ONPO achieves 21.2% and 9.9% relative improvement over the strongest baseline (INPO) with Mistral-Instruct and Llama-3-8B respectively. Academic benchmarks show ONPO preserves base model capabilities without significant alignment tax. The method avoids needing to estimate win rates over the full policy, relying only on binary preference signals.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. No GitHub link, footnote, or supplementary code release is mentioned."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets and models: the RLHFlow prompt collection (huggingface.co/datasets/RLHFlow/prompt-collection-v0.1), RLHFlow/LLaMA3-SFT, Mistral-7B-Instruct-v0.3, and the pair-preference-model. All evaluation benchmarks (AlpacaEval 2.0, Arena-Hard, MT-Bench) are public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment setup details are provided. Appendix B lists training hyperparameters but not software versions, GPU types, or library dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. Algorithm 1 describes the method but there are no runnable scripts or README instructions for reproducing experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only point estimates with no confidence intervals or error bars. Figure 1 shows performance across η values but without uncertainty bands."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are performed. Claims that ONPO 'outperforms' baselines are based solely on comparing point estimates without any t-test, bootstrap, or other significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative improvements with baseline context: '21.2% improvement on Mistral-It' (35.3→42.8 on AlpacaEval) and '9.9% improvement on Llama-3-SFT' (44.2→48.6). Absolute baseline and result values are provided in Table 1."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes. The evaluation benchmarks are standard (805 AlpacaEval instructions, 500 Arena-Hard queries, 80 MT-Bench questions) but no power analysis or discussion of whether these are sufficient for the claimed comparisons."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread across runs is reported anywhere. All results appear to be from single runs with no multi-seed evaluation."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three online baselines are compared: Iterative DPO (Dong et al., 2024), SPPO (Wu et al., 2024), and INPO (Zhang et al., 2024). Additional reference models (Llama-3-70B-it, GPT-4, Claude-3-Opus, etc.) are shown in Table 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All three main baselines (Iterative DPO, SPPO, INPO) are from 2024, representing current state-of-the-art online general preference alignment methods."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The comparison between ONPO and INPO (by the same lead author) effectively ablates the key innovation: INPO uses standard OMD while ONPO adds the optimistic predictor. Section 6.3 provides a hyperparameter sensitivity analysis varying η across 5 values."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three main benchmarks are used: AlpacaEval 2.0 (LC win rate), Arena-Hard (win rate), and MT-Bench (1-10 GPT-4 rating). Table 2 adds six academic benchmarks (GPQA, MMLU-Pro, Hellaswag, Winogrande, TruthfulQA, GSM8K)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation uses GPT-4 Preview-1106 as the automated judge. The paper's core claim is about alignment with human preferences, making human evaluation relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation is on standard held-out benchmarks (AlpacaEval 2.0, Arena-Hard, MT-Bench) that are separate from the training prompt collection (RLHFlow/prompt-collection-v0.1). No tuning is done on the test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-benchmark breakdowns across six academic tasks (GPQA, Hellaswag, MMLU-Pro, Winogrande, TruthfulQA, GSM8K) plus the average. Table 1 breaks down results by benchmark and base model."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases, error analysis, or qualitative examples of where ONPO produces poor outputs are discussed. The paper only shows aggregate performance numbers."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 underlines cases where baselines outperform ONPO (e.g., INPO gets 37.0 vs ONPO's 36.4 on Arena-Hard with Llama-3-SFT), but these are not discussed or analyzed. The text says ONPO 'consistently outperforms or achieves comparable performance' without investigating the losses."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims ONPO 'outperforms state-of-the-art RLHF algorithms across multiple representative benchmarks.' Table 1 shows ONPO wins on 5/6 benchmark-model combinations and is close on the 6th. The O(T⁻¹) theoretical bound is proved in Theorem 4.2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The key causal claim — that the optimistic predictor enables faster convergence — is justified by formal theoretical analysis (Theorem 4.2) proving the O(T⁻¹) bound, plus the controlled comparison to INPO which isolates the optimistic component. The theoretical proof provides adequate causal mechanism."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'LLM General Preference Alignment' but experiments use only two 7-8B parameter models. No testing on larger models, different model families, or non-English settings. The results are specific to single-turn alignment with a particular preference oracle."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The paper doesn't consider whether implementation details, hyperparameter tuning advantages, or the specific preference model choice could explain the results, beyond the theoretical motivation."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures GPT-4 judge win rates and frames this as 'alignment with human preferences' without discussing the gap. The mention that AlpacaEval LC has 0.98 Spearman correlation with Chatbot Arena partially bridges this but the limitations of automated judges as proxies for human preferences are not discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions with HuggingFace URLs are provided: RLHFlow/LLaMA3-SFT (based on Llama-3-8B), Mistral-7B-Instruct-v0.3, RLHFlow/pair-preference-model-LLaMA3-8B, and GPT-4 Preview-1106 as the judge model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided. The training prompt collection is referenced by URL but the prompts themselves are not shown. Judge prompts for AlpacaEval/Arena-Hard/MT-Bench are inherited from those benchmarks but not reproduced."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B reports: cosine learning rate scheduler, peak learning rate 5×10⁻⁷, warm-up ratio 0.03, global batch size 128, 1/η grid search over [0.1, 0.05, 0.02, 0.01, 0.005] with 1/η=0.01 selected, K=8 responses per prompt, 5 iterations for Llama-3-SFT and 3 for Mistral-Instruct."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a training/alignment algorithm paper, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The online training loop is described in Algorithm 1 (sample K=8 responses, tournament selection for yw/yl), but the prompt collection preprocessing is not documented. The paper defers training details to Dong et al. (2024) rather than documenting the full pipeline."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. Section 7 ('Conclusion and Future Work') mentions multi-turn implementation and active data selection as future work, but does not discuss limitations of the current work. The Impact Statement is generic boilerplate."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. No mention of threats from the automated judge, limited model sizes, single preference oracle, or single-run evaluations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries. Section 5.2 notes the multi-turn setting is left for future work, but the paper does not explicitly state what the results do NOT show or which settings are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, preference judgments, per-example scores) is released. Only aggregate benchmark scores are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data generation process is described: at each iteration, K=8 responses are sampled from the current policy using prompts from the RLHFlow prompt collection, then the preference oracle (pair-preference-model) provides binary feedback via tournament selection (Section 6.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks and a public prompt collection."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "Algorithm 1 documents the high-level pipeline, but key details are deferred to Dong et al. (2024), including training details for the preference model, prompt collection construction, and the full experimental workflow."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed. There is no acknowledgments section mentioning grants or sponsors despite authors being from UIUC and Tencent AI Lab."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Illinois Urbana-Champaign and Tencent AI Lab, Bellevue."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Tencent AI Lab, a corporate research lab, employs most of the authors and has commercial interest in RLHF improvements. No funding independence discussion is provided."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff dates for Llama-3-8B and Mistral-7B-v0.3 base models are not stated. Without this, it is impossible to assess whether evaluation benchmarks could be in the pre-training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether AlpacaEval, Arena-Hard, MT-Bench, or academic benchmarks (MMLU-Pro, GPQA, etc.) overlap with the base models' pre-training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Several benchmarks (Hellaswag 2019, Winogrande 2021, GSM8K 2021, TruthfulQA 2021) were published well before Llama-3 and Mistral-v0.3 training. Potential contamination is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example cost is reported. The method requires K=8 response generations plus preference model queries per training iteration, but the cost of this pipeline is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, or total training time are reported. The paper describes hyperparameters (Appendix B) but not the computational resources required."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results are reported. All tables show single point estimates without any indication of seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results appear to be from single runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B states a grid search for 1/η over [0.1, 0.05, 0.02, 0.01, 0.005] (5 values). Figure 1 shows performance across different η values."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B states '1/η = 0.01' was selected but does not specify whether this was chosen on validation or test data, or what selection criterion was used."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 9 benchmarks and 2 base models, but no statistical tests are performed and no correction for multiple comparisons is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of author-evaluation bias. It is unclear whether the authors re-implemented baselines or used original code. The lead author is also the lead author of INPO (Zhang et al., 2024), one of the main baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison of compute budgets across methods. ONPO uses a two-step update per iteration (vs one step for INPO) but the additional compute cost is not quantified or discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether AlpacaEval, Arena-Hard, or MT-Bench actually measure alignment quality. The paper cites the 0.98 Spearman correlation with Chatbot Arena for AlpacaEval but does not question whether these benchmarks capture general preference alignment."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. This is a training algorithm comparison where the same online RLHF workflow is used across methods."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Several academic benchmarks (Hellaswag 2019, GSM8K 2021) predate the base models' training and solutions may be in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The tournament-style preference selection during training could introduce biases not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training prompts and evaluation benchmarks, or whether AlpacaEval/Arena-Hard prompts overlap with the RLHFlow prompt collection."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied. No canary strings, decontamination, or membership inference analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "ONPO achieves an O(T⁻¹) duality gap bound, improving upon the previous O(T⁻¹/²) result of INPO.",
       "evidence": "Theorem 4.2 provides the formal proof with DualGap(π̄) ≤ 4√D/T. The proof in Appendix A.2 leverages the RVU property of optimistic OMD where stability terms cancel out.",
       "supported": "strong"
     },
     {
       "claim": "ONPO achieves 21.2% relative improvement over the strongest baseline on AlpacaEval 2.0 with Mistral-Instruct.",
       "evidence": "Table 1: ONPO achieves 42.8 LC win rate vs INPO's 35.3 on AlpacaEval 2.0 with Mistral-Instruct. (42.8-35.3)/35.3 ≈ 21.2%.",
       "supported": "moderate"
     },
     {
       "claim": "ONPO achieves 9.9% relative improvement over the strongest baseline on AlpacaEval 2.0 with Llama-3-SFT.",
       "evidence": "Table 1: ONPO achieves 48.6 LC win rate vs INPO's 44.2. (48.6-44.2)/44.2 ≈ 9.9%. However, no error bars are reported, so the significance of this difference is unclear.",
       "supported": "moderate"
     },
     {
       "claim": "ONPO consistently outperforms or achieves comparable performance to baselines across both base models.",
       "evidence": "Table 1 shows ONPO wins on 5 of 6 benchmark-model combinations. On Arena-Hard with Llama-3-SFT, INPO scores 37.0 vs ONPO's 36.4 — a small loss that is underlined but not discussed.",
       "supported": "moderate"
     },
     {
       "claim": "ONPO does not over-align the model and preserves intrinsic knowledge and abilities.",
       "evidence": "Table 2 shows ONPO achieves 55.4 average across 6 academic benchmarks with Mistral-Instruct, slightly above the base model (54.6) and all baselines (54.3-55.1). The differences are very small.",
       "supported": "moderate"
     },
     {
       "claim": "ONPO is robust to hyperparameter η variations.",
       "evidence": "Figure 1 shows ONPO outperforms the best baseline across all tested η values on both Arena-Hard and AlpacaEval 2.0. However, only 5 values of 1/η are tested and no error bars are shown.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance across runs",
       "detail": "All results are single point estimates without any uncertainty quantification. For claims of 'outperforming' baselines where margins are sometimes small (e.g., 36.4 vs 37.0 on Arena-Hard), statistical significance is unknown."
     },
     {
       "flag": "No human evaluation for alignment claims",
       "detail": "The paper claims to improve 'alignment with human preferences' but all evaluation uses GPT-4 as an automated judge. No human evaluation is conducted to validate that the improvements transfer to actual human preference."
     },
     {
       "flag": "Self-comparison with lead author's prior work",
       "detail": "The lead author (Yuheng Zhang) is also the lead author of INPO (Zhang et al., 2024), the main baseline being compared against. The implementation details of both methods may not be independently verified."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated discussion of limitations, threats to validity, or scope boundaries. The Impact Statement is generic boilerplate: 'none of which we feel must be specifically highlighted here.'"
     },
     {
       "flag": "Overclaiming in title scope",
       "detail": "The title claims 'LLM General Preference Alignment' but experiments are limited to two 7-8B models, single-turn settings, one preference oracle, and English-only benchmarks. The generality claimed far exceeds what was tested."
     }
   ],
   "cited_papers": [
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["R. Rafailov", "A. Sharma", "E. Mitchell", "C. D. Manning", "S. Ermon", "C. Finn"],
       "year": 2024,
       "relevance": "Foundational offline preference alignment algorithm (DPO) that ONPO builds upon and compares against."
     },
     {
       "title": "Self-play preference optimization for language model alignment",
       "authors": ["Y. Wu", "Z. Sun", "H. Yuan", "K. Ji", "Y. Yang", "Q. Gu"],
       "year": 2024,
       "arxiv_id": "2405.00675",
       "relevance": "Self-play alignment approach (SPPO) that is a direct baseline in the experiments."
     },
     {
       "title": "Iterative nash policy optimization: Aligning llms with general preferences via no-regret learning",
       "authors": ["Y. Zhang", "D. Yu", "B. Peng", "L. Song", "Y. Tian", "M. Huo", "N. Jiang", "H. Mi", "D. Yu"],
       "year": 2024,
       "arxiv_id": "2407.00617",
       "relevance": "Direct predecessor to ONPO (INPO); uses standard OMD for self-play alignment. Main baseline and ablation reference."
     },
     {
       "title": "A general theoretical paradigm to understand learning from human preferences",
       "authors": ["M. G. Azar", "Z. D. Guo", "B. Piot", "R. Munos", "M. Rowland", "M. Valko", "D. Calandriello"],
       "year": 2024,
       "relevance": "First work on general preference alignment (IPO) without BT model assumption, foundational to this line of research."
     },
     {
       "title": "Nash learning from human feedback",
       "authors": ["R. Munos", "M. Valko", "D. Calandriello", "M. G. Azar", "M. Rowland", "Z. D. Guo", "Y. Tang", "M. Geist", "T. Mesnard", "A. Michi"],
       "year": 2023,
       "arxiv_id": "2312.00886",
       "relevance": "First to formulate alignment as a two-player zero-sum game (Nash-MD), the framework ONPO operates within."
     },
     {
       "title": "Direct nash optimization: Teaching language models to self-improve with general preferences",
       "authors": ["C. Rosset", "C.-A. Cheng", "A. Mitra", "M. Santacroce", "A. Awadallah", "T. Xie"],
       "year": 2024,
       "arxiv_id": "2404.03715",
       "relevance": "Alternative general preference alignment method (DNO) compared in the discussion section."
     },
     {
       "title": "RLHF workflow: From reward modeling to online RLHF",
       "authors": ["H. Dong", "W. Xiong", "B. Pang", "H. Wang", "H. Zhao", "Y. Zhou", "N. Jiang", "D. Sahoo", "C. Xiong", "T. Zhang"],
       "year": 2024,
       "arxiv_id": "2405.07863",
       "relevance": "Provides the online RLHF workflow implementation that ONPO follows, including iterative DPO baseline."
     },
     {
       "title": "Human alignment of large language models through online preference optimisation",
       "authors": ["D. Calandriello", "D. Guo", "R. Munos", "M. Rowland", "Y. Tang", "B. Pires", "P. H. Richemond", "C. L. Lan", "M. Valko", "T. Liu"],
       "year": 2024,
       "arxiv_id": "2403.08635",
       "relevance": "Online IPO algorithm for general preference alignment that ONPO is compared against in Section 5."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["L. Ouyang", "J. Wu", "X. Jiang", "D. Almeida", "C. Wainwright", "P. Mishkin"],
       "year": 2022,
       "relevance": "Foundational RLHF work (InstructGPT) establishing the reward model + PPO pipeline that subsequent alignment work builds upon."
     },
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Y. Bai", "A. Jones", "K. Ndousse", "A. Askell"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "RLHF alignment methodology for Claude, demonstrating the practical importance of alignment research."
     },
     {
       "title": "Proximal policy optimization algorithms",
       "authors": ["J. Schulman", "F. Wolski", "P. Dhariwal", "A. Radford", "O. Klimov"],
       "year": 2017,
       "arxiv_id": "1707.06347",
       "relevance": "PPO algorithm widely used in RLHF pipelines that ONPO aims to improve upon in terms of stability and compute."
     },
     {
       "title": "KTO: Model alignment as prospect theoretic optimization",
       "authors": ["K. Ethayarajh", "W. Xu", "N. Muennighoff", "D. Jurafsky", "D. Kiela"],
       "year": 2024,
       "arxiv_id": "2402.01306",
       "relevance": "Alternative alignment algorithm using prospect theory, representing a different approach to preference learning."
     },
     {
       "title": "Multi-turn reinforcement learning from preference human feedback",
       "authors": ["L. Shani", "A. Rosenberg", "A. Cassel", "O. Lang", "D. Calandriello", "A. Zipori"],
       "year": 2024,
       "arxiv_id": "2405.14655",
       "relevance": "Multi-turn RLHF formulation as contextual MDP that ONPO discusses extending to in Section 5.2."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "RLHF practitioners could implement ONPO following Algorithm 1 and the described hyperparameters, but no code is released to make adoption immediate."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Challenges the Bradley-Terry assumption (a known limitation) and improves convergence rate from O(T⁻¹/²) to O(T⁻¹), but the general direction is incremental within established game-theoretic alignment."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or risk concerns raised; this is an alignment improvement paper that aims to make models better aligned."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward algorithmic improvement with standard benchmark comparisons."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or pip-installable tool released; implementation would require significant effort."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Tencent AI Lab is a known corporate lab and UIUC is a respected university, but neither carries the social media attention of OpenAI/Anthropic/Google."
     }
   }
 }