ai-research-survey

scan.json (34062B)

---

 {
   "paper": {
     "title": "RMB: Comprehensively Benchmarking Reward Models in LLM Alignment",
     "authors": [
       "Enyu Zhou",
       "Guodong Zheng",
       "Binghai Wang",
       "Zhiheng Xi",
       "Shihan Dou",
       "Rong Bao",
       "Wei Shen",
       "Limao Xiong",
       "Jessica Fan",
       "Yurong Mou",
       "Rui Zheng",
       "Tao Gui",
       "Qi Zhang",
       "Xuanjing Huang"
     ],
     "year": 2024,
     "venue": "International Conference on Learning Representations (ICLR 2025)",
     "arxiv_id": "2410.09893",
     "doi": "10.48550/arXiv.2410.09893"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states: 'Our evaluation code and datasets are available at https://github.com/Zhou-Zoey/RMB-Reward-Model-Benchmark.' A concrete GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states code and datasets are available at the GitHub URL. The reproducibility statement confirms: 'We will release the code and datasets upon acceptance.' Published at ICLR 2025, so the data should be released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not mention requirements.txt, Dockerfile, or detailed environment setup. Appendix C references evaluation settings but not library versions or dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The reproducibility statement says 'We provide details to reproduce our results in Section 4, Section 5, and Appendix C,' but these describe methodology, not step-by-step instructions with commands. No README or scripts are described in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The main results in Table 3 and all appendix tables (Tables 13-16) report only point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Spearman rank correlation coefficients are computed with p-values reported. Figure 22 shows 'P = 7.22e-9' and 'P = 3.09e-3' for the BoN-pairwise correlation analysis. Section 5 uses Spearman correlation for validation."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are provided in context: 'average decline of 17.5%' for BoN vs pairwise (Section 4.2), correlation of -0.57 between helpfulness/harmlessness rankings (Section 4.2), specific average differences in Table 7, and standard deviations in Table 8."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 18,000+ preference pairs and 3,197 prompts but no justification for why these sizes are sufficient. No power analysis or explicit sample size rationale is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Standard deviations of model scores across models are reported in Table 8, but variance across experimental runs is not reported. For generative RMs using stochastic outputs, no multi-run variance is stated for the main evaluation results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "RewardBench (Lambert et al., 2024) is used as a baseline throughout. Figure 1 and Figure 4 directly compare RMB's correlation with alignment performance against RewardBench's."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "RewardBench (2024) is the 'only existing reward benchmark' and is the most contemporary comparison available. Other datasets compared in Table 1 (Ultrafeedback, Helpsteer2, HH-RLHF) are also recent."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explores impact factors of generative RMs by varying evaluation criteria (original vs verbose) and instructing methods (with/without CoT) in Table 12. They also test majority voting as a confidence-weighting mechanism."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two primary metrics are used: Pairwise Accuracy (Eq. 1) and BoN Accuracy (Eq. 2). Spearman rank correlation with downstream alignment is used for validation. Results are broken down by helpfulness and harmlessness."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Three independent human annotators compared 200 randomly sampled pairs from the dataset and from RewardBench (Table 2). A separate human-annotated held-out set was used to validate the AI feedback algorithm (Table 5, Appendix B.4)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The RMB benchmark is an evaluation set not used for training any of the models being evaluated. A 'separate human-annotated held-out set' was used to validate the scoring process (Section 3.3). The BoN verification uses external benchmarks (MixEval, Arena-Hard, AdvBench)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns are provided: Figure 3 shows per-task performance for helpfulness (12 tasks) and harmlessness (12 scenarios). Appendix H provides full tables (Tables 13-16) for all models across all categories. Figures 27-28 show ranking correlations across tasks."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses where models struggle: 'models tend to perform well in Nonviolent Crime and Specialized Advice scenarios' but 'there is substantial variability in sex-related scenarios, where some models achieve over 70% accuracy, while others fall below random chance levels.' Privacy and Intellectual Property weaknesses are highlighted."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports that majority voting 'may not be effective in the RM evaluation context' (Figure 24). Table 12 shows that 'using more complex helpfulness evaluation criteria tended to reduce performance.' Chain-of-Thought 'did not significantly impact larger models.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 49 scenarios (verified in Appendix A.2), positive correlation with downstream alignment (Section 5, Figure 4), generalization defects revealed (Section 4.3, Figure 3), potential of generative RMs (Table 3). All empirical claims in the abstract have corresponding results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Most claims are correlational (Spearman rank correlation for validation). The ablation in Table 12 varies one factor at a time (criteria complexity, CoT) which is adequate for the causal claims made ('using more complex helpfulness evaluation criteria tended to reduce performance'). Language is appropriately hedged."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded to the tested setting. The title says 'Comprehensively Benchmarking' but the paper clarifies this refers to the HH (helpfulness-harmlessness) alignment goal. Section 5 bounds correlation claims to specific model-benchmark combinations. The limitations section acknowledges scope constraints."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D discusses alternative explanations: intrinsic conflict between objectives (D.2), pitfalls of reward hacking as explanation for trade-off (D.2), and why generative RMs might outperform (D.1 discusses harnessing core generative abilities, alignment with human judgment process, and explicit reasoning). Section 6 considers why majority voting might fail."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes between benchmark accuracy (the proxy) and actual alignment performance (the outcome). Section 5 is entirely devoted to verifying 'Can evaluation results reflect alignment performance?' They use BoN sampling on external benchmarks to test this proxy-outcome relationship."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Most models are specified with versions: 'GPT-4-turbo-2024-04-09' (footnote 2), 'GPT-4o-2024-05-13' (Table 3), 'Claude-3-5-sonnet-20240620' (Table 4), 'Llama-2-70b-chat-hf', 'Mistral-7B-Instruct-v0.1', 'Qwen2-72B-Instruct'. Specific model IDs are given for all evaluated models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided: Figure 15 (two-stage categorization), Figure 16 (helpfulness scoring), Figure 17 (harmlessness scoring), Figure 18 (helpfulness assessment for generative RMs), Figure 19 (harmlessness assessment). Additional prompts in Figures 25-26."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature 0.2 for response generation (Appendix B.2), temperature 1.0 for BoN sampling responses (footnote 5), m=5 for BoN sampling (Section 5.1). Default chat templates used. Appendix C references evaluation settings."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation involves direct scoring by discriminative RMs or pairwise comparison by generative RMs, not multi-step agent workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 and Appendix B.1 describe the full pipeline: pre-filtering by length and language, three-way cross-validation categorization, difficulty post-filtering (removing prompts where weak models performed well). Score gap thresholds (< 2.5) and diversity balancing are documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'LIMITATIONS' section appears after the conclusion, with substantive discussion of four specific limitations including the use of BoN instead of full RL, lack of generative RM validation, insufficient exploration of majority voting, and LLM response diversity concerns."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: (1) BoN used instead of PPO due to RL's 'time-consuming and unstable nature,' (2) generative RM BoN validation omitted because 'AI feedback could lead to bias accumulation,' (3) majority voting not explored in depth with specific hypothesis about filtering effect, (4) LLM response diversity concern with mitigation strategy stated."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The limitations state what was NOT done: 'We did not use a full reinforcement learning process,' 'We did not include validation for generative RMs,' 'we did not explore this in depth.' They also note 'The benchmarks reliance on LLM-generated responses may limit diversity and future adaptability.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our evaluation code and datasets are available at https://github.com/Zhou-Zoey/RMB-Reward-Model-Benchmark.' The full dataset with preference pairs and BoN sets should be available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 details the full data construction process: prompts from WildChat corpus (Section 3.2), response generation from 14 specific LLMs (Table 4, Appendix B.2), GPT-4-based scoring with scenario-specific key features (Section 3.3, Figures 16-17), and preference pair construction with score gap constraints."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "For human annotators: 'The annotators are all undergraduate and graduate students. We paid them a certain wage based on the number of annotations completed' (Appendix B.4). Prompt sources are described (WildChat for help-seeking, red-teaming prompts for harmlessness)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: prompt selection with pre-filtering and categorization → response generation from 14 LLMs → two-stage GPT-4 scoring → preference pair construction with score gap thresholds → BoN set construction. Statistics are provided at each stage (Figures 5-6, Appendix A)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The acknowledgment section lists multiple funding sources: PCL Grant PCL2024A06, NSFC grants (62476061, 62206057, 62076069), Shanghai Rising-Star Program (23QA1400200), Natural Science Foundation of Shanghai (23ZR1403500), and Shanghai Academic Research Leader grant 22XD1401100."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Fudan University (School of Computer Science and Institute of Modern Languages), UNC Chapel Hill, and Pengcheng Laboratory. They evaluate third-party models, not their own."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are Chinese government grants and academic programs (NSFC, Shanghai municipal programs, PCL). None have a financial interest in which reward model performs best on the benchmark."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included. The ethics statement addresses other concerns but does not declare whether any authors have financial interests related to the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates reward models' performance on their benchmark but does not state training data cutoff dates for the evaluated models (GPT-4o, Claude-3.5-sonnet, Starling-RM-34B, etc.). This matters because WildChat is a public corpus some models may have trained on."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether any evaluated reward models were trained on data overlapping with WildChat prompts or the specific response models used. The discriminative RMs in particular are trained on preference data that could share distribution with RMB."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the benchmark is newly constructed (reducing contamination risk), the paper does not explicitly discuss whether the source data (WildChat) or similar preference pairs could have appeared in training data of the evaluated models."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates reward models, not human participants. Human annotators are used for data validation but are not research subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The ethics statement says 'This work does not involve... research with human subjects.' Annotators provided data labels, not studied as participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Annotators are described as 'undergraduate and graduate students' but are not research participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark requires extensive API calls (GPT-4 for scoring, GPT-4o and Claude-3.5-sonnet for evaluation, multiple models for response generation and BoN sampling) but no inference costs are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or API spend is reported despite the extensive use of 14 LLMs for response generation, GPT-4 for scoring, and multiple reward model evaluations."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Generative RM evaluations are stochastic but results are not reported across multiple random seeds. The majority voting experiment (10 iterations) is separate from the main evaluation. Discriminative RM evaluations are deterministic."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The main evaluation does not state the number of runs. The majority voting experiment specifies '10 iterations of voting on two 70B-level LLMs' but this is a separate analysis, not the main evaluation protocol."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The evaluation prompts and settings appear chosen without describing how many configurations were tried or how the final setup was selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation configuration (prompts, scoring criteria, score gap threshold of 2.5) is used without explicit justification for why these specific values were chosen over alternatives. The human-annotated held-out set was used for 'refining' but the selection process is not detailed."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made across 19 models, multiple scenarios, and multiple metrics. The correlation analyses in Figure 4 compare across multiple benchmarks and sampling models. No multiple comparison correction (Bonferroni, etc.) is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors constructed RMB and then evaluate it against RewardBench. They do not acknowledge that as benchmark creators, they may have optimized RMB's design to show favorable correlation with alignment performance. No independent evaluation or bias discussion is provided."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 7B to 72B parameters and API models are compared without discussing compute requirements. An 8B discriminative RM is compared against GPT-4o without noting the massive compute difference."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is entirely devoted to construct validity: verifying that RMB scores correlate with downstream alignment task performance via BoN sampling on MixEval, Arena-Hard, and AdvBench. This directly addresses whether the benchmark measures what it claims."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in reward model evaluation. Models are evaluated directly on preference pairs and BoN lists."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "WildChat data is public and could have been in training data of the evaluated models. The paper does not discuss temporal ordering between benchmark construction and model training. Some reward models may have been trained on similar preference data from the same source models."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For generative RMs seeing both responses simultaneously, position bias is acknowledged but feature leakage through the evaluation format is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether discriminative RMs were trained on preference data sharing distribution with RMB's construction process (e.g., GPT-4-scored preference pairs from similar prompt distributions)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipeline is used."
       }
     }
   },
   "claims": [
     {
       "claim": "RMB demonstrates a stronger positive correlation between evaluation results and RMs' Best-of-N alignment performance compared to RewardBench across multiple external benchmarks.",
       "evidence": "Figure 1 and Figure 4 show Spearman rank correlations between RMB/RewardBench rankings and BoN performance on MixEval, MixEval-Hard, Arena-Hard, and AdvBench. RMB shows positive correlations while RewardBench exhibits poor or negative correlations in several settings (Section 5.2).",
       "supported": "moderate"
     },
     {
       "claim": "Generative models (GPT-4o, Claude-3.5-sonnet) show great promise in reward modeling, surpassing state-of-the-art discriminative RMs.",
       "evidence": "Table 3 shows GPT-4o (0.738 overall), Qwen2-72B (0.723), and Claude-3.5-sonnet (0.706) outperform the best discriminative RM Starling-RM-34B (0.712). GPT-4o tops the overall ranking, and Claude-3.5-sonnet leads on helpfulness.",
       "supported": "strong"
     },
     {
       "claim": "It is hard for an RM to be both competitive in judging helpfulness and harmlessness, with a -0.57 ranking correlation between the two objectives.",
       "evidence": "Table 6 shows the top 10 RMs' separate rankings on helpfulness and harmlessness. Spearman correlation of -0.57 is computed. Eurus-RM-7b ranks 2nd on helpfulness but 9th on harmlessness; GPT-4o ranks 7th on helpfulness but 1st on harmlessness (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "BoN evaluation provides higher difficulty and greater differentiation than pairwise evaluation, with an average 17.5% performance decline.",
       "evidence": "Section 4.2 reports 17.5% average decline from pairwise to BoN. Table 8 shows consistently higher standard deviation for BoN (0.121 vs 0.076 helpfulness, 0.132 vs 0.069 harmlessness). Table 7 breaks down by RM type. Figure 20 visualizes the distribution.",
       "supported": "strong"
     },
     {
       "claim": "Majority voting may not be effective for handling noise in reward model evaluation.",
       "evidence": "Section 6 and Figure 24 show confidence-weighted metrics do not improve correlation between evaluation results and downstream alignment performance. Despite Table 9 showing agreement increases with confidence, the weighted evaluation doesn't enhance benchmark validity.",
       "supported": "moderate"
     },
     {
       "claim": "Using more complex helpfulness evaluation criteria tends to reduce generative RM performance.",
       "evidence": "Table 12 shows verbose criteria hurt most models: Llama3.1-70B (-0.009), Mistral-Large (-0.049), Llama3.1-8B (-0.045), Llama2-70B (-0.081), Qwen2-72B (-0.039). Only Mixtral-8x7B shows marginal improvement (+0.003).",
       "supported": "moderate"
     },
     {
       "claim": "Top RMs show consistent performance across helpfulness scenarios but struggle with diverse harmlessness scenarios.",
       "evidence": "Figure 27 shows helpfulness task correlations mostly >0.85. Figure 28 shows harmlessness correlations with much more variation (e.g., Specialized Advice vs Sexual Content correlation 0.28). Figure 3 confirms visually. Section 4.3 discusses.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RMB is a comprehensive reward model benchmark covering 49 real-world scenarios with both pairwise and Best-of-N evaluation paradigms. The benchmark shows stronger correlation with downstream alignment performance than RewardBench, with generative models (GPT-4o, Claude-3.5-sonnet) outperforming discriminative RMs. A trade-off exists between helpfulness and harmlessness evaluation capability (rank correlation -0.57), and BoN evaluation proves more challenging and discriminative than pairwise testing. Majority voting for confidence weighting did not improve benchmark validity.",
   "red_flags": [
     {
       "flag": "No error bars on main results",
       "detail": "Table 3 and all evaluation tables report single point estimates without confidence intervals, error bars, or multi-run variance. For generative RMs using stochastic decoding, results could vary across runs, but no uncertainty quantification is provided."
     },
     {
       "flag": "Single AI judge for all scoring",
       "detail": "All preference pairs are scored by GPT-4-turbo-2024-04-09 as the sole judge. While human validation shows ~75% agreement, systematic biases in GPT-4's preferences (e.g., favoring verbose responses, certain response styles) could propagate through the entire benchmark. The ~25% disagreement rate means a substantial fraction of labels may not reflect true human preference."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "WildChat is a publicly available corpus. Reward models trained after its release may have seen similar prompts or response patterns in their training data. No contamination analysis is performed despite this being a benchmark evaluating model capabilities."
     },
     {
       "flag": "Correlation validation uses limited model set",
       "detail": "The downstream alignment correlation (Section 5) is computed over a small set of reward models with only a handful of external benchmarks and sampling configurations. Small-N correlations can be unstable, and no bootstrap confidence intervals are provided for the Spearman coefficients."
     }
   ],
   "cited_papers": [
     {
       "title": "RewardBench: Evaluating reward models for language modeling",
       "authors": ["Nathan Lambert", "Valentina Pyatkin", "Jacob Morrison"],
       "year": 2024,
       "arxiv_id": "2403.13787",
       "relevance": "The primary baseline — the only prior reward model benchmark. RMB is positioned as a more comprehensive and alignment-correlated alternative."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF paper defining the alignment process and the role of reward models that RMB evaluates."
     },
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Yuntao Bai", "Andy Jones", "Kamal Ndousse"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "Defines the helpfulness-harmlessness alignment framework that RMB's two evaluation dimensions are based on."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI feedback",
       "authors": ["Yuntao Bai", "Saurav Kadavath", "Sandipan Kundu"],
       "year": 2022,
       "arxiv_id": "2212.08073",
       "relevance": "Introduces AI feedback for alignment, which RMB uses generative models as reward models building on this approach."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2024,
       "relevance": "LLM-as-a-Judge paradigm used for generative RM evaluation in RMB and for benchmark correlation analysis."
     },
     {
       "title": "UltraFeedback: Boosting language models with scaled AI feedback",
       "authors": ["Ganqu Cui", "Lifan Yuan", "Ning Ding"],
       "year": 2024,
       "relevance": "Major preference dataset compared against RMB in Table 1; uses similar AI feedback methodology for data construction."
     },
     {
       "title": "Generative verifiers: Reward modeling as next-token prediction",
       "authors": ["Lunjun Zhang", "Arian Hosseini", "Hritik Bansal"],
       "year": 2024,
       "arxiv_id": "2408.15240",
       "relevance": "Argues for generative approaches to reward modeling, a key finding confirmed by RMB's evaluation results."
     },
     {
       "title": "Secrets of RLHF in large language models part II: Reward modeling",
       "authors": ["Binghai Wang", "Rui Zheng", "Lu Chen"],
       "year": 2024,
       "arxiv_id": "2401.06080",
       "relevance": "Detailed analysis of reward model training dynamics and challenges, directly relevant to understanding RM quality assessment."
     },
     {
       "title": "Self-rewarding language models",
       "authors": ["Weizhe Yuan", "Richard Yuanzhe Pang", "Kyunghyun Cho"],
       "year": 2024,
       "arxiv_id": "2401.10020",
       "relevance": "Proposes using LLMs as their own reward models for alignment, relevant to the generative RM evaluation paradigm in RMB."
     },
     {
       "title": "Llama Guard: LLM-based input-output safeguard for human-AI conversations",
       "authors": ["Hakan Inan", "Kartikeya Upasani", "Jianfeng Chi"],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "Provides the taxonomy of safety categories that RMB's harmlessness scenarios are based on."
     },
     {
       "title": "Helping or herding? Reward model ensembles mitigate but do not eliminate reward hacking",
       "authors": ["Jacob Eisenstein", "Chirag Nagpal", "Alekh Agarwal"],
       "year": 2023,
       "arxiv_id": "2312.09244",
       "relevance": "Studies reward hacking in alignment, directly relevant to RMB's findings about helpfulness-harmlessness trade-offs and reward model limitations."
     },
     {
       "title": "WildChat: 1M ChatGPT interaction logs in the wild",
       "authors": ["Wenting Zhao", "Xiang Ren", "Jack Hessel"],
       "year": 2024,
       "arxiv_id": "2405.01470",
       "relevance": "Primary source of real-world prompts for RMB's benchmark construction."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "RM developers and alignment researchers can directly use the benchmark to evaluate reward models, with code and data released."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Challenges RewardBench's validity and reveals helpfulness-harmlessness trade-off, but these findings are not dramatically surprising to the alignment community."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Reveals reward model weaknesses in harmlessness evaluation, with implications for alignment safety, but does not demonstrate novel attacks."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly critiques RewardBench as showing poor correlation with actual alignment performance, but framed diplomatically as complementary work."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and datasets released on GitHub; researchers can evaluate their own reward models against the benchmark."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From Fudan University; evaluates well-known models (GPT-4o, Claude-3.5-sonnet) but the benchmark itself and authors are not widely recognized."
     }
   }
 }