ai-research-survey

scan.json (30143B)

---

 {
   "paper": {
     "title": "Relative Preference Optimization: Enhancing LLM Alignment through Contrasting Responses across Identical and Diverse Prompts",
     "authors": [
       "Yueqin Yin",
       "Zhendong Wang",
       "Yi Gu",
       "Hai Huang",
       "Weizhu Chen",
       "Mingyuan Zhou"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2402.10958",
     "doi": "10.48550/arXiv.2402.10958"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RPO extends DPO by constructing a contrast matrix that compares preferred/dispreferred responses across semantically related prompts, not just identical ones, using embedding-based similarity reweighting. On Anthropic-HH, OpenAI Summarization, and AlpacaEval2.0, RPO-Paired achieves 78.52% GPT-4 win rate vs DPO's 72.26% on Mistral-7B, and 38.88% on AlpacaEval2.0 vs DPO's 30.84%. Ablations show that naive uniform or diagonal weighting underperforms DPO, confirming that semantic similarity reweighting is the critical component. RPO also works with unpaired preference data, extending applicability beyond paired settings.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract provides a GitHub link: https://github.com/yinyueqin/relative-preference-optimization. Appendix G also includes the core Python implementation of the RPO loss function."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: Anthropic's Helpful and Harmless dataset (170k dialogues) and OpenAI's Summarization dataset. AlpacaEval2.0 is also a public benchmark. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 lists some hyperparameters and mentions '8 Nvidia A100 GPUs' and RMSProp optimizer, but no requirements.txt, Dockerfile, or library versions are provided. Not enough detail to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. A GitHub link is provided and hyperparameters are listed in Table 5, but the paper lacks a 'Reproducing Results' section with specific commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-4 are point estimates (e.g., '78.52' win rate). No confidence intervals, error bars, or ± notation appear anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RPO 'significantly outperforms' baselines (abstract, Section 4.3) but provides no statistical significance tests — no p-values, t-tests, or bootstrap tests. Comparisons are made by simply comparing two numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1-4 report both RPO and baseline win rates, providing context for the magnitude of improvement. For example, RPO-Paired achieves 78.52 vs DPO's 72.26 on Anthropic-HH with Mistral-7B (Table 4), allowing readers to assess the 6.26 percentage point improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 states '256' comparisons for computing win rate, and the ablation studies note '256 samples' from the test set. No justification is given for why 256 is sufficient, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or any spread measures are reported across experimental runs. All results appear to be single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 compares RPO against SFT, PPO, IPO, DPO, and KTO — a comprehensive set of alignment baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include DPO (NeurIPS 2023), IPO (AISTATS 2024), and KTO (2024), all very recent and state-of-the-art methods for preference alignment."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 presents extensive ablation studies: Table 1 ablates weighting strategies (uniform, diagonal, embedding), Table 2 ablates embedding extraction models and temperature, Table 3 ablates batch size. Appendix F adds further ablations on prompt-only vs. prompt-response weighting, beta values, and sampling temperature."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The sole evaluation metric is GPT-4 win rate, applied across different datasets (HH, Summarization, AlpacaEval2.0). While evaluated on multiple tasks, there is only one type of metric — no complementary metrics such as perplexity, BLEU, ROUGE, or human ratings are used."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. The paper uses GPT-4 as the sole evaluator, describing it as 'a stand-in for human evaluators' (Section 4.1)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states evaluations were conducted on 'the validation sets of Anthropic's HH Dataset for dialogue and the OpenAI Summarization Dataset for summarization.' AlpacaEval2.0 uses a separate set of 805 prompts."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 breaks down results by model (LLaMA2-7B, LLaMA2-13B, Mistral-7B) and by task (Anthropic-HH, OpenAI Summarization, AlpacaEval2.0). Ablation tables provide further breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative failure analysis or error analysis is provided. The generation examples in Appendix I show only successful outputs without discussing cases where RPO fails or produces poor responses."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports that Uniform Weighting (68.36) and Diagonal Weighting (69.92) both underperform DPO (72.26). Table 3 shows small batch sizes (2) underperform DPO. Table 6 shows integrated prompt-response weighting reduces generalizability on AlpacaEval."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'superior ability to align LLMs' on dialogue, summarization, and AlpacaEval2.0. Table 4 supports this — RPO-Paired achieves the highest win rates in most settings. The claim about handling 'both paired and unpaired sets' is demonstrated in Table 4 with separate RPO-Unpaired and RPO-Paired results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims (e.g., 'reweighting based on prompt similarities significantly enriches model alignment') are supported by controlled ablation studies. Table 1 systematically varies the weighting strategy while holding other factors constant. Table 2 ablates embedding models, and Table 3 ablates batch size — each is a controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Enhancing LLM Alignment' broadly, but results are limited to 3 models (LLaMA2-7B, LLaMA2-13B, Mistral-7B) on 2 datasets plus AlpacaEval. The abstract's claim of 'superior ability to align LLMs with user preferences' is not bounded to the tested models or tasks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for RPO's improvements. For example, it does not discuss whether the gains could be due to implicit data augmentation (seeing more contrastive pairs per batch), regularization effects of the reweighting, or other confounds beyond the proposed semantic similarity mechanism."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4 win rate as a proxy for human preference alignment. While it acknowledges GPT-4 as 'a stand-in for human evaluators' (Section 4.1), it does not discuss the gap between GPT-4 judgments and actual human preferences, known biases of LLM-as-judge (e.g., verbosity bias, position bias), or limitations of this proxy."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'LLaMA2-7B', 'LLaMA2-13B', and 'Mistral-7B' as base models. Table 5 specifies the GPT-4 judge version as 'gpt-4-0613' and AlpacaEval judge as 'alpaca_eval_gpt4_turbo_fn'. Embedding model specified as 'all-MiniLM-L6-v2'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H provides the full GPT-4 evaluation prompts for both dialogue and summarization tasks. The training data comes from standard datasets, so training prompts are implicitly available."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 in Appendix D provides comprehensive hyperparameters: batch size 64, learning rate 5e-7, 1 epoch, β=0.1, τ values, sampling temperature 0, optimizer (RMSProp), max prompt length 256, max total length 512."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. RPO is a training-time preference optimization method, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the data processing: in paired settings, each batch has N triplets (x, yw, yl); in unpaired settings, triplets are deconstructed into (x, yw) and (x, yl) pairs, shuffled, and N instances of each are sampled. The SFT phase using preferred responses is also described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 includes a 'Limitations & Future Work' subsection identifying three specific limitations: dependency on embedding model quality, contrast matrix limited by single-GPU mini-batch memory, and the constant Z(x) assumption."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The three limitations are specific to RPO: (1) a weak text encoder may fail to capture prompt similarities, (2) contrastive matrix size is bounded by GPU memory within a single mini-batch, and (3) the algorithm assumes constant Z(x) across all prompts. These are not generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not acknowledge that results are limited to 7B-13B models, English-only text, or the specific task domains tested. No explicit boundary statements like 'our results apply only to...' are present."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The underlying training datasets (Anthropic-HH, OpenAI Summarization) are public, but the experimental outputs — model generations, GPT-4 evaluation judgments, and per-sample win/loss decisions — are not released. Only aggregated win rates are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the datasets in detail: Anthropic-HH has '170k dialogues, each comprising a human query and paired model responses rated for helpfulness and harmlessness.' The OpenAI Summarization dataset structure is also described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. All data comes from standard public benchmarks (Anthropic-HH, OpenAI Summarization, AlpacaEval2.0)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents the pipeline: preferred responses used for SFT, then preference pairs processed in paired or unpaired settings. The unpaired construction is explicitly described (deconstruct triplets, shuffle, extract N instances)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments or grant information appears in the paper. Authors are affiliated with UT Austin, Microsoft Azure AI, and Google, but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed on the first page: The University of Texas at Austin, Microsoft Azure AI, and Google."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Authors are affiliated with Microsoft Azure AI and Google, both of which have commercial interests in LLM alignment technology. No funding source is disclosed, making independence unverifiable. The institutional affiliations suggest potential non-independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper. Authors from Microsoft and Google may have relevant financial interests but these are not declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for LLaMA2 or Mistral-7B. These models' pretraining data could include the evaluation datasets (Anthropic-HH published 2022, OpenAI Summarization published 2020), but this is not discussed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between the base models' pretraining data and the evaluation datasets. Since LLaMA2 was released in 2023 and the datasets predate it, contamination is possible."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Both the Anthropic-HH dataset (2022) and OpenAI Summarization dataset (2020) were publicly available before the training cutoffs of LLaMA2 (2023) and Mistral (2023). This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper evaluates training methods using automated benchmarks and GPT-4 as judge."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported. The additional computational overhead of computing prompt embeddings and the contrast matrix is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '8 Nvidia A100 GPUs' (Table 5) but does not state total GPU hours, training wall-clock time, or total compute budget for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of running experiments with multiple random seeds. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never explicitly stated. Results are presented as single values without indicating whether they are from one run or averaged over multiple."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The ablation studies explore several hyperparameter values (τ, embedding models, batch sizes, β values, sampling temperatures), but the total computational budget for this search is not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The configuration selection is justified through systematic ablation. Table 2 shows all-MiniLM-L6-v2 at τ=0.5 was selected based on performance on Anthropic-HH. Table 1 justifies embedding reweighting over alternatives. Section 4.2 explains the reasoning."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across methods, models, datasets, and hyperparameter settings without any correction for multiple comparisons (Bonferroni, Holm, etc.)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines themselves and compare RPO against these implementations. No discussion of potential author-evaluation bias or reference to independent baseline implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "RPO introduces additional compute overhead from embedding computation and contrast matrix construction compared to DPO, but performance is not reported as a function of compute budget. No matched-compute comparison is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4 win rate as the sole evaluation metric without discussing whether it actually measures alignment with human preferences. It cites Zheng et al. (2023) and Li et al. (2023) to justify GPT-4 as evaluator but does not discuss construct validity limitations."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. RPO is a training-time method and all comparisons use the same inference setup."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation datasets (Anthropic-HH 2022, OpenAI Summarization 2020) were publicly available before the base models (LLaMA2, Mistral) were trained in 2023. This temporal leakage risk is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setup or features could leak answer information. The GPT-4 evaluation prompts in Appendix H show both responses simultaneously, but position bias is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap or structural similarity between training and test splits of the datasets used."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied — no canary strings, membership inference, decontamination, or temporal analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "RPO with embedding reweighting significantly outperforms DPO on dialogue tasks",
       "evidence": "Table 1 shows RPO Embedding Reweighting (Paired, τ=0.5) achieves 78.52% win rate vs DPO's 72.26% on Anthropic-HH with Mistral-7B. Table 4 shows consistent improvements across LLaMA2-7B (68.75 vs 63.67) and LLaMA2-13B (72.66 vs 63.28).",
       "supported": "moderate"
     },
     {
       "claim": "Semantically related prompts serve as effective contrastive pairs only with proper reweighting",
       "evidence": "Table 1 shows Uniform Weighting (68.36) and Diagonal Weighting (69.92) both underperform DPO (72.26), while Embedding Reweighting surpasses it (78.52). This ablation demonstrates that naive cross-prompt contrast is harmful but similarity-weighted contrast helps.",
       "supported": "moderate"
     },
     {
       "claim": "RPO works with both paired and unpaired preference data",
       "evidence": "Table 4 shows RPO-Unpaired achieves competitive results: 75.00 on Anthropic-HH (vs DPO's 72.26), 50.39 on Summarization (vs DPO's 48.83), and 31.24 on AlpacaEval2.0 (vs DPO's 30.84). In some settings RPO-Unpaired matches or exceeds RPO-Paired.",
       "supported": "moderate"
     },
     {
       "claim": "Larger batch sizes improve RPO performance",
       "evidence": "Table 3 shows monotonic improvement from batch size 2 (71.48%) to 8 (78.52%) per GPU on Anthropic-HH with Mistral-7B.",
       "supported": "moderate"
     },
     {
       "claim": "RPO outperforms all state-of-the-art alignment methods across tasks",
       "evidence": "Table 4 shows RPO-Paired achieves the highest win rates in 4 of 5 settings. However, on OpenAI Summarization, RPO-Unpaired (50.39) and RPO-Paired (50.00) are close to the baseline, and all results lack error bars or significance tests.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "GPT-4 as sole evaluator",
       "detail": "All results rely exclusively on GPT-4 win rate with no human evaluation. Known biases of LLM-as-judge (verbosity preference, position bias) are not discussed. The paper's core claim about 'alignment with human preferences' is evaluated without any human involvement."
     },
     {
       "flag": "No error bars or variance reporting",
       "detail": "All results across Tables 1-4 and Appendix F are single-run point estimates with no standard deviations, confidence intervals, or multiple-seed results. The word 'significantly' is used in claims without any statistical significance testing."
     },
     {
       "flag": "Small evaluation sample size",
       "detail": "Win rates are computed on only 256 samples (Table 5), which is relatively small. With no variance reporting, it is unclear whether the observed differences (e.g., 78.52 vs 72.26) are statistically meaningful or within sampling noise."
     },
     {
       "flag": "Potential conflicts of interest undisclosed",
       "detail": "Authors are affiliated with Microsoft Azure AI and Google, both companies with significant commercial interests in LLM alignment technology. No funding, competing interests, or financial interest statements appear in the paper."
     },
     {
       "flag": "Claims of 'significant' improvement without significance tests",
       "detail": "The abstract and Section 4.3 use language like 'significantly outperforms' and 'superior ability' without any statistical tests. The observed improvements could be within noise of single-run experiments."
     }
   ],
   "cited_papers": [
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell", "Stefano Ermon", "Christopher D Manning", "Chelsea Finn"],
       "year": 2023,
       "relevance": "The foundational method that RPO extends; core baseline for LLM alignment without reward models."
     },
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Yuntao Bai", "Andy Jones", "Kamal Ndousse"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "Source of the Anthropic-HH dataset used for evaluation; foundational work on RLHF alignment."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF work establishing the reward-model-based alignment pipeline that DPO and RPO seek to improve upon."
     },
     {
       "title": "A general theoretical paradigm to understand learning from human preferences",
       "authors": ["Mohammad Gheshlaghi Azar", "Mark Rowland", "Bilal Piot"],
       "year": 2024,
       "relevance": "Proposes IPO (Identity Preference Optimization), a key baseline for preference alignment that addresses DPO overfitting."
     },
     {
       "title": "KTO: Model alignment as prospect theoretic optimization",
       "authors": ["Kawin Ethayarajh", "Winnie Xu", "Niklas Muennighoff", "Dan Jurafsky", "Douwe Kiela"],
       "year": 2024,
       "arxiv_id": "2402.01306",
       "relevance": "Proposes KTO for unpaired preference optimization using prospect theory; key baseline and concurrent work to RPO."
     },
     {
       "title": "Proximal policy optimization algorithms",
       "authors": ["John Schulman", "Filip Wolski", "Prafulla Dhariwal", "Alec Radford", "Oleg Klimov"],
       "year": 2017,
       "arxiv_id": "1707.06347",
       "relevance": "PPO is the standard RL algorithm used in RLHF pipelines; baseline method for LLM alignment."
     },
     {
       "title": "Llama 2: Open foundation and fine-tuned chat models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "One of the base LLMs used for evaluation (LLaMA2-7B, LLaMA2-13B)."
     },
     {
       "title": "Mistral 7b",
       "authors": ["Albert Q Jiang", "Alexandre Sablayrolles", "Arthur Mensch"],
       "year": 2023,
       "arxiv_id": "2310.06825",
       "relevance": "Base LLM used for evaluation; the model on which RPO shows its strongest results."
     },
     {
       "title": "Learning to summarize with human feedback",
       "authors": ["Nisan Stiennon", "Long Ouyang", "Jeffrey Wu"],
       "year": 2020,
       "relevance": "Source of the OpenAI Summarization dataset used for evaluation; foundational RLHF work for summarization."
     },
     {
       "title": "Smaug: Fixing failure modes of preference optimisation with dpo-positive",
       "authors": ["Arka Pal", "Deep Karkhanis", "Samuel Dooley", "Manley Roberts", "Siddartha Naidu", "Colin White"],
       "year": 2024,
       "arxiv_id": "2402.13228",
       "relevance": "Addresses failure modes in DPO-based preference optimization, relevant to understanding DPO limitations that RPO also targets."
     },
     {
       "title": "Judging llm-as-a-judge with mt-bench and chatbot arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "arxiv_id": "2306.05685",
       "relevance": "Foundational work on using LLMs as evaluators, the methodology RPO relies on for all its evaluation results."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "RPO is a drop-in replacement for DPO that practitioners training LLMs could implement; code is provided on GitHub."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The idea of cross-prompt contrastive pairs is novel but not deeply contrarian — it extends DPO in an intuitive direction."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; this is a preference alignment improvement method."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or drama; straightforward methodological contribution."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is on GitHub but requires substantial compute (8 A100s) to train; not a pip-installable tool or demo."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from UT Austin, Microsoft, and Google — recognized institutions but not flagship AI lab releases."
     }
   }
 }