ai-research-survey

scan.json (29886B)

---

 {
   "paper": {
     "title": "TODO: Enhancing LLM Alignment with Ternary Preferences",
     "authors": [
       "Yuxiang Guo",
       "Lu Yin",
       "Bo Jiang",
       "Jiaqi Zhang"
     ],
     "year": 2024,
     "venue": "International Conference on Learning Representations (ICLR 2025)",
     "arxiv_id": "2411.02442",
     "doi": "10.48550/arXiv.2411.02442"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "TODO extends the Bradley-Terry model to handle ties in preference data (TOBT), enabling a ternary preference optimization algorithm that consistently outperforms DPO on Mistral-7B and Llama 3-8B across preference modeling accuracy, MT Bench, and six standard benchmarks. The method benefits from a margin parameter α that improves robustness to noisy preference labels, and performs well even on purely binary preference data without any tie labels. Optimal performance is typically achieved at 10-30% tie data ratios, though the best ratio varies across models and benchmarks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'The implementation details and datasets can be found in https://github.com/XXares/TODO', providing a public GitHub repository URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims datasets are available at the GitHub URL. Additionally, the primary datasets used (Ultrafeedback, Chatarena) are publicly available, and the paper provides HuggingFace links for model checkpoints and data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports training hyperparameters (Table 10) but does not specify library versions, Python version, PyTorch version, CUDA version, or provide a requirements.txt or Dockerfile."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a GitHub link but does not include step-by-step reproduction instructions, a 'Reproducing Results' section, or specific commands to run in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All result tables (Tables 2-5, 7-8, 12-13) and figures (Figures 2-3) report only point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims TODO 'outperforms' and 'consistently outperforms' DPO but provides no statistical significance tests (no p-values, t-tests, or bootstrap tests). All comparisons are based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2-5 provide absolute performance numbers for all methods alongside baselines (SFT, DPO), allowing direct computation of effect magnitudes. For example, Table 2 shows DPO at 66.24 average vs TODO at 71.02 at tie ratio 0.0."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The training set size of 20k and test set size of 1500 are stated without justification or power analysis for why these sizes were chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance measures, or results across multiple runs are reported anywhere in the paper. All results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "DPO is the primary baseline throughout. Section 6.4 extends comparisons to KTO, SimPO, and ODPO (Tables 4-5). SFT-only baselines are also included in Tables 2-3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baselines are recent: DPO (2023), KTO (2024), SimPO (2024), ODPO (2024). These represent contemporary alignment methods."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper varies the tie data ratio (0, 0.1, 0.2, 0.3) to isolate the effect of tie data. Appendix A.3 (Tables 7-8) varies α across {0.1, 0.2, 0.5, 0.8, 1.2} to study its sensitivity. Section 6.3 analyzes key factors."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses preference test set accuracy, Reward Bench accuracy (5 subcategories), MT Bench scores, and six standard benchmarks (Piqa, ARC-c, ARC-e, MMLU, Hellaswag, Winogrande)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated. MT Bench uses GPT-4-turbo as judge (not humans). Other benchmarks use automated metrics via OpenCompass. No human evaluation of model outputs is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes a curated 1500-sample in-distribution test set separate from training data. Reward Bench serves as an external out-of-distribution test set. Standard benchmarks (MMLU, etc.) are also separate."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Reward Bench results are broken down by category (Chat, ChatHard, Safety, Reasoning, Prior) in Appendix A.12 (Tables 12-13). Individual benchmark results are shown separately in Tables 2-3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Tables 7-8 in Appendix A.3 show that TODO diverges at α=0.8 and α=1.2 in several settings. Table 2 shows TODO at tie ratio 0.3 (67.37) degrades below the SFT baseline (69.33) for Mistral."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports divergence at high α values (Tables 7-8), and performance degradation at 30% tie ratio for Mistral (Table 2, TODO drops from 71.02 to 67.37). These are genuine negative results showing method limitations."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of consistent outperformance over DPO are supported by Figures 2-3 and Tables 2-5. Claims about Mistral-7B and Llama 3-8B are verified. The claim about binary preference alignment is supported by results at tie ratio 0.0."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('TODO improves alignment') are supported by controlled experiments varying one factor at a time (tie ratio, method) while keeping all other hyperparameters consistent (Section 5.2). The α ablation in Appendix A.3 provides additional controlled evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Enhancing LLM Alignment' suggests general applicability, but results are only on two 7-8B models. The paper does not explicitly bound its claims to these model sizes, architectures, or language settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider whether the margin α alone (without proper tie handling) could explain the gains on binary data. TODO at tie ratio 0.0 already outperforms DPO, suggesting the margin effect is significant, but this is not discussed as an alternative explanation."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures benchmark accuracy and MT Bench scores but frames these as demonstrating improved 'alignment.' No discussion of whether these proxy metrics actually capture alignment quality, or the gap between benchmark performance and genuine human preference alignment."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers with HuggingFace URLs are provided: alignment-handbook/zephyr-7b-sft-full, kykim0/llama3-8b-ultrachat-sft. For evaluation: 'gpt-4-turbo-2024-04-09' and 'gpt-4o-2024-05-13' are specified with version dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "For MT Bench evaluation with GPT-4 as judge, no actual prompts or scoring instructions are provided. Appendix A.10 says 'we use the default prompt template' from OpenCompass without reproducing the actual text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 10 (Appendix A.9) lists learning rate, batch size, and β for both models. Additional hyperparameters stated: α=0.5, 3 epochs, Adam optimizer, weight decay 0, cosine learning rate scheduler."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper proposes a training loss function (TODO), not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 and Appendix A.8 describe sampling 20k from Ultrafeedback's 383k pairs, maintaining source distribution (Table 9), and classifying ties based on quality score equality. Tie ratio construction at {0, 0.1, 0.2, 0.3} is explained."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has no dedicated 'Limitations' or 'Threats to Validity' section. Section 7 (Discussion) discusses future integration possibilities. Section 8 (Conclusion) mentions future work but does not discuss limitations of TODO itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no discussion of potential confounds, generalizability concerns, or methodological limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No discussion of limitations from testing on only 7-8B models, primarily English tasks, or specific dataset families."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The base datasets (Ultrafeedback, Chatarena) are publicly available with HuggingFace links provided. The paper states constructed datasets are available at the GitHub repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 and Appendix A.8 describe the data source (Ultrafeedback with 383k pairs), sampling procedure, tie classification criteria (identical quality scores from GPT-4), and source distribution (Table 9)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public preference datasets (Ultrafeedback, Chatarena)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: Ultrafeedback (383k pairs, scored by GPT-4) → sample 20k maintaining source distribution → classify ties by quality score equality → construct datasets at tie ratios {0, 0.1, 0.2, 0.3}. Appendix A.8 provides distribution details."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. Work was done at Meituan Inc. (major Chinese tech company) but no explicit funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Meituan Inc., Beihang University, and University of Surrey. The footnotes note which work was done during internship/research at Meituan."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Meituan Inc. (an industry employer of multiple authors) could benefit from improved alignment methods, representing a potential non-independent interest."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for the base Mistral-7B or Llama 3-8B pre-trained models, which is relevant since the evaluation benchmarks may have been in the pre-training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the pre-trained models' training data overlaps with evaluation benchmarks (MMLU, ARC, Piqa, Hellaswag, Winogrande), all of which were publicly available before these models were trained."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation benchmarks like MMLU (2021), ARC (2018), Piqa (2019), and Hellaswag (2019) were all published well before Mistral-7B (2023) and Llama 3 (2024) training. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments are automated model training and benchmark evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study trains and evaluates LLMs using existing preference datasets."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or API cost information is reported for the trained models or the GPT-4-based evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, total API spend, or training time are reported despite fine-tuning 7-8B parameter models."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be single-run numbers without any seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without clarifying whether they represent single runs or averages."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "While Appendix A.3 shows α sensitivity analysis across 5 values, no formal hyperparameter search budget (total compute, number of configurations tried for other hyperparameters) is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.3 provides theoretical and empirical justification for α=0.5 selection, showing loss tradeoffs (Figure 5), performance across α values (Tables 7-8), and threshold-based criteria for loss balance."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed at all, so the question of multiple comparison correction does not arise."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement TODO and compare against their own implementations of other methods. Self-comparison bias is not acknowledged or mitigated."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "TODO and DPO differ only in their loss function; compute differences are negligible. This comparison is not meaningful here."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses MMLU, ARC, Piqa, Hellaswag, Winogrande, and MT Bench to demonstrate 'alignment' but does not discuss whether these benchmarks actually measure alignment quality or what their construct validity limitations are."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. TODO is a training loss function, not an agentic scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Evaluation benchmarks (MMLU 2021, ARC 2018, Piqa 2019, Hellaswag 2019) existed well before model pre-training cutoffs."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setups leak information. The perplexity-based evaluation mode (PPL) for multiple-choice tasks is standard but its potential information leakage properties are not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between the training data (Ultrafeedback) and evaluation benchmarks. Potential overlap between Ultrafeedback's source data and benchmark test items is not considered."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "TODO consistently outperforms DPO in preference modeling accuracy on both in-distribution and out-of-distribution datasets across varying tie data proportions.",
       "evidence": "Figure 2 shows higher accuracy for TODO on the 1500-sample test set and Reward Bench across tie ratios {0, 0.1, 0.2, 0.3} for both Mistral-7B and Llama 3-8B. Detailed Reward Bench subcategory scores in Tables 12-13 (Appendix A.12).",
       "supported": "moderate"
     },
     {
       "claim": "TODO outperforms DPO on MT Bench across all training sets for both Mistral and Llama 3 models.",
       "evidence": "Figures 3a and 3b show TODO scores higher than DPO at all four tie ratios for both model families. Mistral: TODO 6.74 vs DPO 6.46 at ratio 0.0; Llama 3: TODO 6.91 vs DPO 6.73 at ratio 0.2.",
       "supported": "moderate"
     },
     {
       "claim": "TODO achieves better average performance than DPO on six standard benchmarks (Piqa, ARC-c, ARC-e, MMLU, Hellaswag, Winogrande) across all tie data ratios.",
       "evidence": "Tables 2 and 3 show TODO outperforms DPO at every tie ratio on average. Mistral: TODO ranges 67.37-71.17 vs DPO 65.57-70.78. Llama 3: TODO ranges 70.86-71.13 vs DPO 70.42-70.75.",
       "supported": "moderate"
     },
     {
       "claim": "TODO shows strong results in binary preference alignment (without tie data), outperforming DPO even at tie ratio 0.0.",
       "evidence": "Tables 2-3 at tie ratio 0.0: Mistral TODO 71.02 vs DPO 66.24; Llama 3 TODO 71.03 vs DPO 70.75. MT Bench: Mistral TODO 6.74 vs DPO 6.46; Llama 3 TODO 6.74 vs DPO 6.69.",
       "supported": "moderate"
     },
     {
       "claim": "Strategic incorporation of tie data enhances alignment capabilities when used with TODO.",
       "evidence": "Best performance on six benchmarks: Mistral at 20% tie ratio (71.17), Llama 3 at 30% (71.13). MT Bench: Llama 3 peaks at 20% tie ratio (6.91). However, Mistral MT Bench peaks at 0% tie ratio (6.74), and 30% tie ratio degrades Mistral significantly (67.37).",
       "supported": "weak"
     },
     {
       "claim": "TODO outperforms other binary alignment methods (KTO, SimPO, ODPO) when tie data is present.",
       "evidence": "Tables 4-5 compare methods on Ultrafeedback and Chatarena datasets. With tie data, TODO achieves highest test accuracy (78.47 on Chatarena) and MT Bench (5.96 on Ultrafeedback with ties, 5.83 on Chatarena with ties).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or statistical tests",
       "detail": "All comparisons across 12+ experimental conditions are based solely on comparing point estimates from apparently single runs. Many improvements are small (e.g., <1% on Llama 3 benchmarks in Table 3), making it impossible to distinguish signal from noise without variance estimates."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations, threats to validity, or scope boundaries discussion. For an ICLR paper, this is a notable omission that obscures the method's known weaknesses and boundaries."
     },
     {
       "flag": "Inconsistent optimal tie ratio",
       "detail": "The 'optimal' tie data ratio varies across models and benchmarks: Mistral MT Bench peaks at 0%, Mistral six-benchmarks at 20%, Llama 3 MT Bench at 20%, Llama 3 six-benchmarks at 30%. This undermines the practical guidance about how much tie data to use."
     },
     {
       "flag": "Missing compute and resource information",
       "detail": "No GPU hours, hardware specifications, or training times are reported for fine-tuning 7-8B parameter models, making practical cost comparison impossible."
     },
     {
       "flag": "Margin effect confounded with tie handling",
       "detail": "TODO at tie ratio 0.0 (no tie data) already outperforms DPO, suggesting the margin α alone provides substantial benefit. The paper does not disentangle the contribution of the margin from the tie-handling mechanism, making it unclear which component drives the gains."
     }
   ],
   "cited_papers": [
     {
       "title": "Direct Preference Optimization: Your Language Model is Secretly a Reward Model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell", "Stefano Ermon", "Christopher D. Manning", "Chelsea Finn"],
       "arxiv_id": "2305.18290",
       "relevance": "Core baseline method — DPO is the foundational binary preference alignment algorithm that TODO extends."
     },
     {
       "title": "KTO: Model Alignment as Prospect Theoretic Optimization",
       "authors": ["Kawin Ethayarajh", "Winnie Xu", "Niklas Muennighoff", "Dan Jurafsky", "Douwe Kiela"],
       "year": 2024,
       "relevance": "Alternative alignment method compared against TODO; uses prospect theory instead of BT model for preference optimization."
     },
     {
       "title": "SimPO: Simple Preference Optimization with a Reference-Free Reward",
       "authors": ["Yu Meng", "Mengzhou Xia", "Danqi Chen"],
       "year": 2024,
       "arxiv_id": "2405.14734",
       "relevance": "Reference-free alignment baseline compared against TODO in Section 6.4."
     },
     {
       "title": "Direct Preference Optimization with an Offset",
       "authors": ["Afra Amini", "Tim Vieira", "Ryan Cotterell"],
       "year": 2024,
       "arxiv_id": "2402.10571",
       "relevance": "ODPO introduces reward-difference margin to DPO; compared against TODO and discussed as integration candidate in Section 7."
     },
     {
       "title": "Proximal Policy Optimization Algorithms",
       "authors": ["John Schulman", "Filip Wolski", "Prafulla Dhariwal", "Alec Radford", "Oleg Klimov"],
       "arxiv_id": "1707.06347",
       "relevance": "Foundational RL algorithm used in RLHF alignment pipelines that TODO aims to improve upon."
     },
     {
       "title": "Zephyr: Direct Distillation of LM Alignment",
       "authors": ["Lewis Tunstall", "Edward Beeching", "Nathan Lambert"],
       "arxiv_id": "2310.16944",
       "relevance": "Provides the Ultrafeedback-binarized dataset and zephyr-sft-full model used as the Mistral baseline in TODO experiments."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI Feedback",
       "authors": ["Yuntao Bai", "Saurav Kadavath", "Sandipan Kundu"],
       "year": 2022,
       "relevance": "Foundational work on AI feedback for alignment, relevant to preference data quality and LLM-as-judge approaches used in TODO's training data."
     },
     {
       "title": "ORPO: Monolithic Preference Optimization without Reference Model",
       "authors": ["Jiwoo Hong", "Noah Lee", "James Thorne"],
       "arxiv_id": "2403.07691",
       "relevance": "Reference-free alignment alternative that eliminates the reference model requirement from DPO-style methods."
     },
     {
       "title": "RewardBench: Evaluating Reward Models for Language Modeling",
       "authors": ["Nathan Lambert", "Valentina Pyatkin", "Jacob Morrison"],
       "arxiv_id": "2403.13787",
       "relevance": "Out-of-distribution evaluation benchmark used to assess TODO's preference modeling accuracy."
     },
     {
       "title": "UltraFeedback: Boosting Language Models with High-Quality Feedback",
       "authors": ["Ganqu Cui", "Lifan Yuan", "Ning Ding"],
       "year": 2023,
       "relevance": "Primary preference dataset used for TODO training and evaluation; contains GPT-4-scored response pairs with quality scores enabling tie identification."
     },
     {
       "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference",
       "authors": ["Wei-Lin Chiang", "Lianmin Zheng", "Ying Sheng"],
       "year": 2024,
       "relevance": "Human-labeled preference platform providing the Chatarena dataset used for TODO evaluation; contains 29.4% naturally occurring ties."
     },
     {
       "title": "Insights into Alignment: Evaluating DPO and its Variants Across Multiple Tasks",
       "authors": ["Amir Saeidi", "Shivanshu Verma", "Chitta Baral"],
       "arxiv_id": "2404.14723",
       "relevance": "Evaluates DPO variants across tasks; provides context for understanding alignment method performance differences."
     },
     {
       "title": "Direct Alignment of Language Models via Quality-Aware Self-Refinement",
       "authors": ["Runsheng Yu", "Yong Wang", "Xiaoqi Jiao", "Youzhi Zhang", "James T. Kwok"],
       "year": 2024,
       "arxiv_id": "2405.21040",
       "relevance": "Concurrent work using intrinsic knowledge constraints for preference alignment, discussed as compatible with TODO in Section 7."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners doing RLHF/DPO alignment can adopt TODO as a drop-in loss function replacement; code is released on GitHub."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The idea that ties in preference data matter is reasonable but unsurprising; it's an incremental extension to the well-known BT model rather than a paradigm shift."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the paper is about improving alignment quality."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or dramatic claims; a straightforward methodological improvement paper."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub with datasets; implementable by those with GPU access for fine-tuning 7-8B models."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Meituan is a major Chinese tech company but not globally prominent in AI research; published at ICLR which carries prestige."
     }
   }
 }