ai-research-survey

scan.json (30552B)

---

 {
   "paper": {
     "title": "RLTHF: Targeted Human Feedback for LLM Alignment",
     "authors": [
       "Yifei Xu",
       "Tusher Chakraborty",
       "Emre Kıcıman",
       "Bibek Aryal",
       "Eduardo Rodrigues",
       "Srinagesh Sharma",
       "Roberto Estevao",
       "Maria Angels de Luis Balaguer",
       "Jessica Wolk",
       "Rafael Padilha",
       "Leonardo Nunes",
       "Shobana Balakrishnan",
       "Songwu Lu",
       "Ranveer Chandra"
     ],
     "year": 2025,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2502.13417",
     "doi": "10.48550/arXiv.2502.13417"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RLTHF, a human-AI hybrid framework, achieves preference accuracy comparable to fully human-annotated datasets (89.6% vs 91.8% on HH-RLHF, 88.0% vs 89.6% on TL;DR) using only 6-7% of total human annotation effort. Models fine-tuned with DPO on RLTHF-curated data outperform those trained on fully human-annotated data (58.1% vs 55.7% win rate on HH-RLHF). Ablation studies show that both human annotation amplification and back-off sanitization are essential components, and pure self-improvement without human feedback fails to exceed the initial AI baseline.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL, GitHub link, or archive is provided anywhere in the paper, appendix, or footnotes."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: HH-RLHF (Bai et al., 2022a) with 161K samples and TL;DR (Völske et al., 2017) with 93K samples filtered by OpenAI (Stiennon et al., 2020). Both are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix D.2 mentions '8×A100 NVIDIA GPUs with DeepSpeed' but provides no software versions, no requirements.txt, no Dockerfile, and no library version details beyond model names."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. Algorithm 1 in Appendix B gives pseudocode but not executable instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Figures 3-5 and Tables 1-2 are reported as single point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'RLTHF outperforms Random', 'RLTHF outperforms Human baseline') but no statistical significance tests are reported anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are provided with baseline context throughout: e.g., '74.7% to 89.6% with only 6% human annotations' (Section 4.1.1), '15.9× and 5.3× higher ROI' (Section 4.1.2), and win rates in Table 2 with all baselines shown for comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the choice of dataset sizes, shard sizes, or number of iterations. The 1/4 shard is selected as 'optimal' from empirical observation but no power analysis or formal justification is given."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across seeds, or spread measures are reported. All results appear to be from single experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared: (1) AI-only labeling (GPT-4o / GPT-4o mini), (2) Random human annotation with equal annotation budget, and (3) full human annotation. Described in Section 4 and Appendix D.3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines use GPT-4o and GPT-4o mini (2024 models), and the comparison against full human annotation is standard. The RLAIF-style baseline is current."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.4 presents ablation studies: removing human annotations entirely ('No Annotation'), and removing both amplification and back-off mechanisms ('No Ampl./Back-off'). Results in Table 1 show both are necessary."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two evaluation metrics are used: RM preference accuracy on test data (Section 4.1) and pairwise win rate on downstream DPO-trained models via AlpacaEval (Section 4.2, Table 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs is conducted. Win rates are computed using Claude 3.5 Sonnet as an LLM judge (Section 4.2, Appendix E). The 'human annotations' in the paper are for training data labels, not output evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.1.1 states 'All test samples are completely separated from the training samples throughout the experiments.' Section 4.2 confirms 'Evaluations are conducted on held-out test sets' of 4K samples each."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (HH-RLHF vs TL;DR), by shard size (full, 1/2, 1/4, 1/8), by initial LLM (GPT-4o vs GPT-4o mini), and by iteration number. Hyperparameter effects are shown per-setting in Figure 5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 discusses suboptimal configurations: excessive amplification (α=8) leads to overfitting, and excessive down-sampling (1/8 shard) limits accuracy. Table 1 shows the failure of self-improvement alone."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 ablation shows 'No Annotation' achieves no improvement (stays at ~75% for HH-RLHF). Section 4.1.3 reports that α=8 and α=1 are suboptimal, and further subdividing annotation batches 'does not provide a meaningful advantage.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'full-human annotation-level alignment with only 6-7%' matches Figure 3 (89.6% vs 91.8% on HH-RLHF with 6%); 'outperform those trained on fully human-annotated datasets' matches Table 2 (58.1% vs 55.7% on HH-RLHF)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'amplification and back-off are essential' are supported by controlled ablation in Table 1, which manipulates single variables. The ablation design (removing components one at a time) is adequate for these internal component claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract bounds claims to 'HH-RLHF and TL;DR datasets' specifically. The paper doesn't claim universal applicability, and Section 3.2.5 notes that 'effectiveness depends on factors such as hyperparameter tuning, the original data distribution, and model selection.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations for the main results. The paper does not consider whether improvements might be due to data selection effects independent of the reward distribution approach, or whether the label-flipping ground truth procedure biases results in favor of RLTHF."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures RM preference accuracy and LLM-judged win rate but frames these as 'alignment' without discussing the gap. Win rate via Claude 3.5 Sonnet is a proxy for human preference; the paper acknowledges self-enhancement bias (Appendix E) but does not discuss whether LLM-judged win rates reliably measure actual alignment with human preferences."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o' and 'GPT-4o mini' without API version or snapshot dates. 'Llama-3.1-8B-Instruct' and 'Qwen2.5-3B' are specified by size but without exact checkpoint versions. 'Claude 3.5 Sonnet' is used as judge without a version identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix A: system prompt (A.1), user prompt for HH-RLHF (A.2), user prompt for TL;DR (A.3), and adapted win-rate prompts in Appendix E (E.1, E.2). These include actual text, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.2 reports detailed hyperparameters: SFT (lr=2e-5, warmup=0.2, batch=32, 4 epochs), RM (lr=1e-4, LoRA rank=32, alpha=64, batch=128, 2 epochs), DPO (lr=1e-6, beta=0.1/0.5, batch=64, 4 epochs). RLTHF-specific configs in Appendix D.4."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. RLTHF is a reward modeling and annotation pipeline, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.1 describes data preparation including dataset composition. Appendix D.1.2 details the label-flipping procedure with specific percentages (25% for HH-RLHF, 20% for TL;DR) and justification via Table 3. Sharding approach is described in Section 3.2.5."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The paper has only a brief 'Impact Statement' that says 'There are many potential societal consequences of our work, none which we feel must be specifically highlighted here.' This does not constitute substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. Appendix D.4 briefly notes hyperparameters 'may underestimate RLTHF's full potential' but this is not a threat-to-validity discussion."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what settings, tasks, or conditions RLTHF would not work for, or what the results do not show."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The underlying datasets (HH-RLHF and TL;DR) are publicly available. However, the RLTHF-specific intermediate outputs (reward distributions, iteration-level datasets) are not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.1.1 describes both datasets: HH-RLHF (161K samples, conversation context with preferred/non-preferred responses) and TL;DR (93K samples, Reddit posts with human preference pairs on model summarizations)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No new human participants were recruited. The 'human annotations' used in experiments are drawn from existing labels in the public HH-RLHF and TL;DR datasets. Data sources are standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: Figure 1 gives an overview, Algorithm 1 (Appendix B) provides pseudocode, Section 3 describes each stage in detail, and Appendix D describes experimental configuration. The label-flipping transformation is documented with counts (25%/20%)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or grants are disclosed. The paper notes 'Work done during an internship at Microsoft' but does not include an acknowledgments section listing funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: 13 authors at Microsoft, 1 at UCLA (first author, who did the work during a Microsoft internship). The Microsoft affiliation is clearly stated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Microsoft, the employer of all authors, has a direct commercial interest in reducing RLHF annotation costs for its Azure AI and Copilot products. The funder is not independent of the research outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (GPT-4o, GPT-4o mini, Llama-3.1-8B-Instruct, Qwen2.5-3B). HH-RLHF (2022) and TL;DR (2017/2020) predate all these models, raising contamination concerns for the initial AI labeling step."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4o may have been trained on HH-RLHF or TL;DR data, which could inflate the initial AI labeling accuracy and bias the RLTHF starting point."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HH-RLHF was published in 2022 and TL;DR in 2017/2020, well before the training of GPT-4o, Llama 3.1, and Qwen 2.5. No discussion of whether these benchmarks appeared in any model's training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. 'Human annotations' are drawn from existing labels in public datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses pre-existing public preference datasets."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F provides detailed cost analysis: GPT-4o costs $0.003/sample, GPT-4o mini $0.00018/sample. RM training costs $32.77/hour on 8×A100. Table 5 summarizes total costs for different configurations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F states RM training and inference takes '<8 hours on the full dataset, and less than 2 hours on the 1/4 subset' per iteration, with 7 iterations. Hardware: 8×A100 80GB node. Total costs in Table 5."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be from single experimental runs with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never explicitly stated. Results are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1.3 varies hyperparameters systematically, but Appendix D.4 states configurations are 'chosen based on heuristics and limited empirical observations' with no total search budget reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 1/4 shard is selected as 'optimal' from Figure 3 by visual inspection. RLTHF hyperparameters (α, β schedules) are chosen by heuristics. No formal validation-set-based selection procedure is described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparative claims across multiple conditions, datasets, and iterations without performing any statistical tests, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines (AI-only, Random annotation) themselves and compare against their own method. No acknowledgment of author-evaluation bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Performance is not reported as a function of compute budget. RLTHF uses iterative RM training (additional compute) but this is not compared against baselines at matched compute levels."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HH-RLHF and TL;DR preference accuracy actually measures alignment quality, or whether these benchmarks are valid proxies for real-world alignment goals."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. RLTHF is a data curation and reward modeling pipeline."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HH-RLHF (2022) and TL;DR (2017/2020) were published well before the training of GPT-4o, Llama 3.1, and Qwen 2.5. No discussion of whether models may have seen these preference labels during pre-training, which could inflate initial AI labeling performance."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks answer information. The reward model is trained and evaluated on the same dataset distribution without addressing potential information leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While 'all test samples are completely separated from the training samples' (Appendix D.1.1), there is no discussion of structural similarities between train and test splits (e.g., same Reddit threads, similar conversation patterns)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. No canary strings, membership inference tests, or decontamination procedures are used."
       }
     }
   },
   "claims": [
     {
       "claim": "RLTHF achieves full-human annotation-level alignment with only 6-7% of human annotation effort",
       "evidence": "Figure 3 shows RLTHF on 1/4 shard reaches 89.6% preference accuracy on HH-RLHF (vs 91.8% full human) with 6% annotation, and 88.0% on TL;DR (vs 89.6% full human) with 7% annotation (Section 4.1.1).",
       "supported": "moderate"
     },
     {
       "claim": "Models trained on RLTHF-curated datasets outperform those trained on fully human-annotated datasets on downstream tasks",
       "evidence": "Table 2 shows DPO-trained Qwen2.5-3B win rates: RLTHF(4o) achieves 58.1% vs Human 55.7% on HH-RLHF, and 62.3% vs 60.2% on TL;DR. Evaluated using Claude 3.5 Sonnet as judge (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "RLTHF achieves 15.9× higher return on investment in human annotation compared to random annotation on HH-RLHF",
       "evidence": "Section 4.1.2 reports 'With just 6% human annotation, RLTHF (4o) achieves a 15.9× and 5.3× higher ROI compared to Random (4o) on HH-RLHF and TL;DR, respectively.' Supported by Figure 4.",
       "supported": "moderate"
     },
     {
       "claim": "Self-improvement without human annotations fails to exceed the initial AI labeling baseline",
       "evidence": "Table 1 ablation: 'No Annotation' condition stays at ~75% for HH-RLHF and ~75% for TL;DR across all iterations, while full RLTHF improves to 87.7% and 83.7% respectively (Section 4.1.4).",
       "supported": "strong"
     },
     {
       "claim": "Both amplification and back-off mechanisms are essential for RLTHF's effectiveness",
       "evidence": "Table 1: 'No Ampl./Back-off' condition shows only marginal improvement (75.8% after 5 iterations on HH-RLHF vs 87.7% for full RLTHF). Section 4.1.4 confirms both mechanisms are necessary.",
       "supported": "strong"
     },
     {
       "claim": "RLTHF is robust to weaker initial AI labeling quality",
       "evidence": "Figure 4 and Table 2: GPT-4o mini starts with 2.6%/5.6% accuracy gap vs GPT-4o on HH-RLHF/TL;DR, but after 10% annotation the gap shrinks to 0.4%/-0.2%. Table 2 win rates are comparable (Section 4.1.2).",
       "supported": "moderate"
     },
     {
       "claim": "RLTHF reduces overall annotation cost by 84-86%",
       "evidence": "Appendix F cost analysis: full human annotation costs $5,788.80 vs RLTHF(4o) at $926.70 and RLTHF(4o mini) at $813.30, including LLM and compute overhead (Table 5).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own method",
       "detail": "All 14 authors are affiliated with Microsoft (one also with UCLA). Microsoft has commercial interest in reducing RLHF annotation costs for its Azure AI and Copilot fine-tuning services. No independent evaluation is included."
     },
     {
       "flag": "No error bars or statistical tests",
       "detail": "All comparisons across Figures 3-5 and Tables 1-2 are point estimates from apparently single runs. No standard deviations, confidence intervals, or significance tests are reported despite numerous comparative claims."
     },
     {
       "flag": "Manipulated ground truth via label flipping",
       "detail": "Appendix D.1.2: The authors train an RM on all human labels, then flip 25% (HH-RLHF) and 20% (TL;DR) of labels that disagree with their RM, using the result as 'ground truth.' This means ground truth is partially defined by a model, and RLTHF is then evaluated on its ability to recover this model-influenced ground truth. This introduces potential circularity."
     },
     {
       "flag": "Simulated human-in-the-loop",
       "detail": "The 'targeted human feedback' in experiments is drawn from existing dataset labels, not from actual human annotators providing feedback in real time. The paper's claims about human annotation efficiency assume this simulation faithfully represents real annotation workflows, which is unvalidated."
     },
     {
       "flag": "LLM-as-judge for downstream evaluation",
       "detail": "Win rates in Table 2 are judged by Claude 3.5 Sonnet, not human evaluators. The paper mitigates self-enhancement bias by using a different model family but does not validate whether LLM judge rankings correlate with actual human preferences for these specific tasks."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no discussion of limitations, scope boundaries, or threats to validity. The Impact Statement dismisses societal consequences entirely."
     }
   ],
   "cited_papers": [
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Yuntao Bai"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "Foundational RLHF work and source of the HH-RLHF dataset used in this paper's experiments."
     },
     {
       "title": "Learning to summarize with human feedback",
       "authors": ["Nisan Stiennon"],
       "year": 2020,
       "relevance": "Seminal work on RLHF for summarization, source of the TL;DR preference dataset used in experiments."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov"],
       "year": 2024,
       "relevance": "DPO is a core component of RLTHF's downstream training pipeline and one of the main fine-tuning methods evaluated."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang"],
       "year": 2022,
       "relevance": "InstructGPT paper establishing RLHF as the standard approach for aligning LLMs with human preferences."
     },
     {
       "title": "RLAIF: Scaling reinforcement learning from human feedback with ai feedback",
       "authors": ["Harrison Lee"],
       "year": 2023,
       "relevance": "Directly addresses the AI feedback alternative to human annotation that RLTHF builds upon and aims to improve."
     },
     {
       "title": "Self-evolved reward learning for LLMs",
       "authors": ["Chenghua Huang"],
       "year": 2024,
       "arxiv_id": "2411.00418",
       "relevance": "Self-improvement approach for reward learning that RLTHF's ablation shows fails without human intervention."
     },
     {
       "title": "Self-rewarding language models",
       "authors": ["Weizhe Yuan"],
       "year": 2024,
       "arxiv_id": "2401.10020",
       "relevance": "LLM self-improvement paradigm that RLTHF contrasts with by showing human feedback is essential."
     },
     {
       "title": "Is DPO superior to PPO for LLM alignment? A comprehensive study",
       "authors": ["Shusheng Xu"],
       "year": 2024,
       "arxiv_id": "2404.10719",
       "relevance": "Comparative study of preference optimization methods relevant to RLTHF's DPO-based downstream training."
     },
     {
       "title": "Secrets of RLHF in large language models part II: Reward modeling",
       "authors": ["Binghai Wang"],
       "year": 2024,
       "arxiv_id": "2401.06080",
       "relevance": "Analysis of reward modeling challenges including annotation noise and biases that RLTHF addresses."
     },
     {
       "title": "Proximal policy optimization algorithms",
       "authors": ["John Schulman"],
       "year": 2017,
       "arxiv_id": "1707.06347",
       "relevance": "PPO is one of the two main RL algorithms RLTHF supports for downstream LLM fine-tuning."
     },
     {
       "title": "A critical evaluation of AI feedback for aligning large language models",
       "authors": ["Archit Sharma"],
       "year": 2024,
       "arxiv_id": "2402.12366",
       "relevance": "Evaluates limitations of AI feedback for alignment, motivating RLTHF's targeted human annotation approach."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI feedback",
       "authors": ["Yuntao Bai"],
       "year": 2022,
       "arxiv_id": "2212.08073",
       "relevance": "Foundational RLAIF approach that RLTHF extends by incorporating strategic human corrections."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners doing RLHF could apply the targeted annotation strategy to reduce human labeling costs by 84-86%, though no code is released."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that RLTHF outperforms full human annotation is mildly surprising, but the general idea that targeted annotation beats random annotation is expected."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the paper focuses on annotation efficiency for alignment."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims about existing methods or organizations."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code repository, demo, or installable tool is provided."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "From Microsoft Research, published at ICML 2025; uses GPT-4o prominently in experiments."
     }
   }
 }