ai-research-survey

scan.json (24437B)

---

 {
   "paper": {
     "title": "AlphaPO: Reward Shape Matters for LLM Alignment",
     "authors": [
       "Aman Gupta",
       "Shao Tang",
       "Qingquan Song",
       "Sirou Zhu",
       "Jiwoo Hong",
       "Ankan Saha",
       "Viral Gupta",
       "Noah Lee",
       "Eunki Kim",
       "Siyu Zhu",
       "Parag Agrawal",
       "Natesh Pillai",
       "S. Sathiya Keerthi"
     ],
     "year": 2025,
     "venue": "ICML 2025",
     "arxiv_id": "2501.03884"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code release link is provided in the paper. The paper references SimPO's GitHub for hyperparameters but does not release AlphaPO's own code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: UltraFeedback (Cui et al., 2024), AlpacaEval 2.0, and Arena-Hard. The models used are publicly available on HuggingFace (Table 4 lists HuggingFace IDs)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '8×A100 GPUs with the adamw_torch optimizer based on the alignment-handbook' (Appendix A.6) but does not provide a requirements.txt, Dockerfile, or detailed library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. While hyperparameters are listed in Table 3 and Appendix A.6, there are no runnable commands or structured reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 in Appendix A.7 reports 95% confidence intervals for Arena-Hard results (e.g., '95 CI high' and '95 CI low' columns). AlpacaEval 2 results include standard deviation (STD) columns."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AlphaPO outperforms SimPO and DPO across multiple benchmarks but does not use any statistical significance tests (no p-values, t-tests, or bootstrap tests). Comparisons are based solely on point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative improvements with baseline context, e.g., 'AlphaPO leads to about 7% to 10% relative improvement in alignment performance for the instruct versions of Mistral-7B and Llama3-8B while achieving 15% to 50% relative improvement over DPO' (Abstract). Table 1 provides absolute numbers enabling effect size computation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why the specific benchmarks (805 prompts in AE2, Arena-Hard) are sufficient for the claims being made. No power analysis or sample size discussion."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports standard deviation (STD) for AlpacaEval 2 win rates across instructions (e.g., STD 1.40% for AlphaPO on Mistral). Arena-Hard reports 95% confidence intervals."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares AlphaPO against DPO and SimPO across all experimental settings (Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "SimPO (Meng et al., 2024) and DPO (Rafailov et al., 2023) are the state-of-the-art DAA methods. The paper also compares with SPPO (Wu et al., 2024). These are contemporary and competitive baselines."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablation studies on the effect of alpha (Figure 3 middle, Figure 8) and gamma (Figure 3 left, Figure 9) on AE2 performance. Section 4.2 systematically varies these parameters."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates on AlpacaEval 2.0 (length-controlled win rate and raw win rate), Arena-Hard (win rate), HellaSwag, and TruthfulQA (Table 2). Response length is also tracked."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated: AlpacaEval 2.0 uses GPT-4 as judge, Arena-Hard uses automated comparison, and HellaSwag/TruthfulQA are automated benchmarks. The paper discusses 'human preferences' and 'human evaluations' conceptually but does not conduct any."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation benchmarks (AlpacaEval 2.0 with 805 prompts, Arena-Hard) are separate from the training data (UltraFeedback). HellaSwag and TruthfulQA are also independent test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides results broken down by model (Llama-3, Mistral, Gemma-2), by reward model (PairRM vs ArmoRM), and by benchmark. Table 5 provides further per-model detail. Win rate heatmaps in Appendix A.8 show per-instance comparisons."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses that for Gemma-2, AlphaPO has slightly lower or comparable WR on AE2 and AH compared to SimPO (Section 4.2). The drop-off for large negative alpha values is discussed. The paper notes GPT-4's bias toward longer responses as a confound."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that AlphaPO does not improve over SimPO on Gemma-2 Arena-Hard (57.7 vs 59.0, Table 1). It also reports that large negative alpha values hurt performance (Figure 3, Figure 8), and discusses that f-DPO fails to improve with non-zero alpha (Section 3.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '7% to 10% relative improvement' over SimPO and '15% to 50% relative improvement over DPO' for Mistral-7B and Llama3-8B. Table 1 supports these: e.g., Llama-3 LC goes from 42.05 (SimPO) to 45.37 (AlphaPO), about 7.9% relative; Mistral LC goes from 29.71 to 33.03, about 11.2% relative. DPO improvements are also confirmed."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about the effect of alpha on alignment performance. These are supported by controlled single-variable ablations (varying alpha while fixing other hyperparameters, Section 4.2, Figure 3) and theoretical gradient analysis (Theorems 3.1, 3.2, Corollary 3.3)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Reward Shape Matters for LLM Alignment' makes a broad claim. Results are limited to three model families (7B-9B scale), one training dataset (UltraFeedback), and two evaluation benchmarks. The paper does not explicitly bound the generalization to these specific settings, nor discuss whether results hold for larger models or different domains."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why AlphaPO outperforms SimPO beyond the reward shape analysis. For instance, it does not consider whether the improvement could stem from the additional hyperparameter search (alpha adds a new tuning dimension) rather than the reward shape itself. No threats-to-validity section addresses confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 in Appendix A.6 provides exact HuggingFace model IDs: 'mistralai/Mistral-7B-Instruct-v0.2', 'meta-llama/Meta-Llama-3-8B-Instruct', 'google/gemma-2-9b-it'. These are specific, versioned identifiers."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting in the conventional sense. It trains models using preference optimization and evaluates on standard benchmarks. The prompts come from the benchmark datasets (AlpacaEval 2.0, Arena-Hard), not from the authors."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 (Appendix A.6) reports alpha, beta, gamma/beta, and learning rate for all models and methods. Additional details include batch size (128), max sequence length (2048), cosine learning rate schedule with warmup ratio 0.1, and decoding temperatures for evaluation."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a standard preference optimization method for LLM training."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the data preprocessing: 'we regenerate five responses for every prompt in the UF dataset using a sampling temperature of 0.8. We then use two reward models - PairRM and ArmoRM to rank the 5 responses. The highest scoring response is labeled yw and the lowest scoring response is labeled yl.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper has an 'Impact Statement' (Section after Conclusion) but it only states 'There are many potential societal consequences of our work, none which we feel must be specifically highlighted here.' This is not a limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. There is no consideration of confounds, evaluation bias, or limitations of the experimental setup."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its claims to the tested model sizes, the UltraFeedback dataset, or the specific evaluation benchmarks used."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw experimental outputs (model generations, per-prompt scores) are not made available. Only aggregate results in tables and figures are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes how training data was created: regenerating responses from the UltraFeedback dataset using sampling temperature 0.8, then ranking with PairRM or ArmoRM reward models."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Data comes from standard public benchmarks and automated response generation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from UltraFeedback prompts through response regeneration, reward model ranking, to preferred/dispreferred pair selection is clearly documented in Section 4.1. The evaluation pipeline uses standard AlpacaEval 2.0 and Arena-Hard tooling."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The paper lists LinkedIn Corporation as an affiliation but does not have an acknowledgments section disclosing funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: LinkedIn Corporation (most authors) and KAIST AI. The footnote notes some authors' work was 'done while at LinkedIn Corporation.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "LinkedIn Corporation (a Microsoft subsidiary) has a direct commercial interest in LLM alignment methods. The majority of authors are LinkedIn employees. This creates a potential non-independent funding relationship, though the paper evaluates open-source models rather than LinkedIn's proprietary models."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for Mistral-7B-Instruct-v0.2, Llama-3-8B-Instruct, or Gemma-2-9B-it. Since these models are being further fine-tuned, the pre-training cutoff is relevant to understanding potential benchmark contamination."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether AlpacaEval 2.0 or Arena-Hard prompts/answers could have appeared in the pre-training data of the base models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "AlpacaEval was published in 2023 and HellaSwag in 2019, before the training cutoffs of the models used. No contamination analysis is provided. Arena-Hard is relatively newer but still no contamination discussion."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported for the trained models. The paper does not mention API costs or tokens consumed during evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.6 states: 'All the training experiments in this paper were conducted on 8×A100 GPUs... The training time for Mistral-Instruct and Llama-3-Instruct is around 2.3 hours, while Gemma-2-Instruct requires 3 hours.'"
       }
     }
   },
   "claims": [
     {
       "claim": "AlphaPO achieves 7% to 10% relative improvement in length-controlled win rate over SimPO on AlpacaEval 2.0 for Llama3-8B and Mistral-7B instruct models.",
       "evidence": "Table 1 shows Llama-3: 45.37 vs 42.05 LC (7.9% relative), Mistral: 33.03 vs 29.71 LC (11.2% relative) with PairRM data.",
       "supported": "strong"
     },
     {
       "claim": "AlphaPO achieves 15% to 50% relative improvement over DPO on the same models.",
       "evidence": "Table 1 shows Llama-3: 45.37 vs 39.18 LC (15.8% relative), Mistral: 33.03 vs 21.96 LC (50.4% relative) with PairRM data.",
       "supported": "strong"
     },
     {
       "claim": "The reward function shape significantly impacts preference-based alignment learning, with the alpha parameter controlling the intensity of likelihood displacement.",
       "evidence": "Theoretical analysis in Theorems 3.1 and 3.2, supported by training dynamics experiments in Figures 2 and 7 showing non-monotonic behavior of likelihood displacement with varying alpha.",
       "supported": "strong"
     },
     {
       "claim": "A slightly positive value of alpha achieves the best alignment performance.",
       "evidence": "Figure 3 (middle) and Figure 8 show peak AE2 performance at alpha=0.25 for Mistral and Llama-3, and alpha=0.1 for Gemma-2.",
       "supported": "strong"
     },
     {
       "claim": "AlphaPO combined with SPPO achieves 47.42% length-controlled win rate on AlpacaEval 2.0.",
       "evidence": "Section 4.2 reports AlphaPO+SPPO achieves 47.42% LC, improved from 45.37% for AlphaPO alone, using PairRM-based UltraFeedback data.",
       "supported": "moderate"
     },
     {
       "claim": "AlphaPO outperforms SimPO on HellaSwag and TruthfulQA benchmarks.",
       "evidence": "Table 2 shows marginal improvements: HellaSwag 0.7694 vs 0.7576 (Llama-3) and 0.8638 vs 0.8610 (Mistral); TruthfulQA 0.6142 vs 0.6078 (Llama-3) and 0.7127 vs 0.7061 (Mistral). Improvements are small and no statistical tests are applied.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "AlphaPO introduces a parameterized reward function shape (via an alpha parameter applied to alpha-divergence with length normalization) for direct alignment algorithms. Theoretical gradient analysis and empirical experiments demonstrate that the reward shape controls likelihood displacement intensity in a non-monotonic fashion. AlphaPO outperforms SimPO by 7-10% and DPO by 15-50% on AlpacaEval 2.0 length-controlled win rate for Llama-3-8B and Mistral-7B, though gains are smaller or absent for Gemma-2-9B. The method adds one hyperparameter (alpha) to SimPO's existing framework.",
   "red_flags": [
     {
       "flag": "No significance testing",
       "detail": "All comparisons between AlphaPO, SimPO, and DPO are based on point estimates without any statistical significance tests. The improvements on HellaSwag and TruthfulQA (Table 2) are small enough that they could be within noise."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations section. The 'Impact Statement' is boilerplate with no substantive discussion. No threats to validity, scope boundaries, or alternative explanations are provided."
     },
     {
       "flag": "Additional hyperparameter confound",
       "detail": "AlphaPO introduces alpha as an additional tunable hyperparameter. The paper does not discuss whether the improvement could partly stem from having an additional dimension of hyperparameter search rather than the reward shape itself. SimPO has 3 hyperparameters; AlphaPO has 4."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Evaluation benchmarks (AlpacaEval, HellaSwag, TruthfulQA) predate the models' training. No analysis of whether benchmark data leaked into pre-training corpora."
     },
     {
       "flag": "Single-run results for main benchmarks",
       "detail": "The main results in Table 1 appear to be from single best runs per hyperparameter configuration. While Table 5 reports STD across instructions, there is no report of variance across training seeds or runs."
     }
   ],
   "cited_papers": [
     {
       "title": "Direct Preference Optimization: Your Language Model is Secretly a Reward Model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell", "Christopher D. Manning", "Stefano Ermon", "Chelsea Finn"],
       "year": 2023,
       "relevance": "Foundational DAA method that AlphaPO builds upon and compares against."
     },
     {
       "title": "SimPO: Simple Preference Optimization with a Reference-Free Reward",
       "authors": ["Yu Meng", "Mengzhou Xia", "Danqi Chen"],
       "year": 2024,
       "relevance": "Primary baseline; AlphaPO generalizes SimPO's log reward function with an alpha-parameterized family."
     },
     {
       "title": "Beyond Reverse KL: Generalizing Direct Preference Optimization with Diverse Divergence Constraints",
       "authors": ["Chaoqi Wang", "Yibo Jiang", "Chenghao Yang", "Han Liu", "Yuxin Chen"],
       "year": 2024,
       "relevance": "f-DPO method that inspired AlphaPO's alpha-divergence reward, but without length normalization."
     },
     {
       "title": "Unintentional Unalignment: Likelihood Displacement in Direct Preference Optimization",
       "authors": ["Noam Razin", "Sadhika Malladi", "Adithya Bhaskar", "Danqi Chen", "Sanjeev Arora", "Boris Hanin"],
       "year": 2024,
       "arxiv_id": "2410.08847",
       "relevance": "Key prior work on likelihood displacement phenomenon that AlphaPO addresses."
     },
     {
       "title": "Self-Play Preference Optimization for Language Model Alignment",
       "authors": ["Yue Wu", "Zhiqing Sun", "Huizhuo Yuan", "Kaixuan Ji", "Yiming Yang", "Quanquan Gu"],
       "year": 2024,
       "arxiv_id": "2405.00675",
       "relevance": "SPPO method shown to be complementary to AlphaPO for further alignment improvements."
     },
     {
       "title": "ORPO: Monolithic Preference Optimization without Reference Model",
       "authors": ["Jiwoo Hong", "Noah Lee", "James Thorne"],
       "year": 2024,
       "relevance": "Introduced length normalization for reward in preference optimization, a key ingredient in AlphaPO."
     },
     {
       "title": "Scaling Laws for Reward Model Overoptimization in Direct Alignment Algorithms",
       "authors": ["Rafael Rafailov", "Yaswanth Chittepu", "Ryan Park", "Harshit Sikchi", "Joey Hejna", "Bradley Knox", "Chelsea Finn", "Scott Niekum"],
       "year": 2024,
       "relevance": "Studies reward over-optimization in DAAs, a phenomenon AlphaPO aims to mitigate."
     },
     {
       "title": "Understanding Likelihood Over-Optimisation in Direct Alignment Algorithms",
       "authors": ["Zhengyan Shi", "Simon Land", "Acyr Locatelli", "Matthieu Geist", "Max Bartolo"],
       "year": 2024,
       "relevance": "Studies over-optimization in DAAs showing that controlled likelihood reduction benefits generalization."
     },
     {
       "title": "Tulu 3: Pushing Frontiers in Open Language Model Post-Training",
       "authors": ["Nathan Lambert"],
       "year": 2024,
       "arxiv_id": "2411.15124",
       "relevance": "Major open-source post-training effort that adopted SimPO's length normalization, demonstrating its practical impact."
     },
     {
       "title": "f-PO: Generalizing Preference Optimization with f-Divergence Minimization",
       "authors": ["Jiayi Han", "Mingjian Jiang", "Yansong Song", "Jure Leskovec", "Stefano Ermon", "Minkai Xu"],
       "year": 2024,
       "arxiv_id": "2410.21662",
       "relevance": "Related f-divergence approach to preference optimization addressing over-optimization."
     },
     {
       "title": "UltraFeedback: Boosting Language Models with Scaled AI Feedback",
       "authors": ["Ganqu Cui", "Lifan Yuan", "Ning Ding"],
       "year": 2024,
       "relevance": "Primary training dataset used in all AlphaPO experiments."
     }
   ]
 }