ai-research-survey

scan.json (22585B)

---

 {
   "paper": {
     "title": "Bayesian Reward Models for LLM Alignment",
     "authors": [
       "Adam X. Yang",
       "Maxime Robeyns",
       "Thomas Coste",
       "Zhengyan Shi",
       "Jun Wang",
       "Haitham Bou Ammar",
       "Laurence Aitchison"
     ],
     "year": 2024,
     "venue": "Structured Probabilistic Inference & Generative Modeling workshop, ICML 2024",
     "arxiv_id": "2402.13210",
     "doi": "10.48550/arXiv.2402.13210"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL is provided in the paper. No GitHub link, Zenodo archive, or any other code release is mentioned."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available AlpacaFarm dataset (Dubois et al., 2024) for both SFT and reward model training. The Pythia model suite is also publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification, requirements.txt, Dockerfile, or detailed dependency list is provided. The paper mentions using Pythia models and LoRA but does not specify software versions or dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or reproduction scripts are provided. The experimental details in Appendix C describe hyperparameters but not how to run the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Figures 2-5 show only point curves without confidence intervals, error bars, or uncertainty bands. No ± notation or CI values appear in the text."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Laplace-LoRA 'effectively mitigate[s] reward overoptimization' and that LA Ens 'demonstrated further enhancements' over ensembles, but no statistical significance tests (p-values, bootstrap tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are shown only as curves in figures (proxy/gold reward vs. KL divergence). No numerical effect sizes, percentage improvements, or baseline-contextualized differences are reported in the text."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 1000 prompts and 12,500 responses per prompt for BoN sampling (Section 5) but provides no justification for why these numbers were chosen or whether they are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread across experimental runs is reported. Results appear to be single-run without any indication of variability across seeds or initializations."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against the standard MAP (single reward model) baseline and reward model ensembles (Ens) across all experiments (Figures 2-5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include reward model ensembles (Coste et al., 2024; Eisenstein et al., 2023) and weight averaging (Ramé et al., 2024), which are contemporary approaches from 2023-2024."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares variance-based vs. standard deviation-based penalties (Figures 2a vs 2b, 3a vs 3b), tests multiple values of the hyperparameter k (1, 5, 10, 20, 30), and compares single LA vs. LA combined with ensembles."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are evaluated using both proxy reward and gold reward scores (left and right columns of all figures), which measure different aspects of performance."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses a synthetic evaluation setup with a gold reward model as the ground truth. No human evaluation of the selected responses is conducted, though the system's goal is to improve alignment with human preferences."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the AlpacaFarm instructions validation dataset (a separate split from the training data) for BoN evaluation (Section 5: 'we collect a subset of 1000 prompts from the AlpacaFarm instructions validation dataset')."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are shown only as aggregate curves across all prompts. No per-category or per-task breakdowns are provided (e.g., by prompt difficulty, domain, or response length)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D discusses failure cases: larger penalty values (k=20, 30) degrade performance for standard deviation-based penalty more significantly, while variance-based penalty is more robust. The paper also shows the failure mode of MAP (overoptimization) in Figure 1."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that larger k values (20, 30) degrade performance for the standard deviation-based penalty (Appendix D), and notes the conclusion section mentions RLHF results but acknowledges they are only briefly shown."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'the resulting uncertainty estimates can effectively mitigate reward overoptimization in BoN sampling.' The figures show improved gold reward scores at high KL divergence for Laplace-LoRA methods, supporting this claim within the synthetic evaluation framework."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about uncertainty penalties mitigating overoptimization. The ablation design (varying k, comparing variance vs. std penalties, single vs. ensemble) provides controlled single-variable manipulations that adequately support these claims within the experimental framework."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and conclusion make broad claims about mitigating reward overoptimization generally, but experiments use only a single synthetic setup (Pythia 70M proxy, LLaMA 7B gold, AlpacaFarm data). The title 'Bayesian Reward Models for LLM Alignment' suggests generality that is not supported by the limited experimental setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements. For example, it does not consider whether the penalty simply constrains optimization (a simpler mechanism than Bayesian uncertainty) or whether other forms of regularization might achieve similar effects."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies exact model architectures: Pythia 1.4B for the LLM policy, Pythia 70M for the proxy reward model, and LLaMA 7B for the gold reward model (Section 5). These are specific named models from known suites with known parameter counts."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The prompt format with special tokens (<|prompter|>, <|assistant|>, <|endoftext|>) is provided in Appendix C.1 (Table 2) with a concrete example."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed hyperparameter tables are provided in Appendix C: Table 3 for SFT (learning rate, scheduler, batch size, epochs) and Table 4 for reward model training (LoRA r, alpha, dropout, weight decay, learning rate, scheduler, batch size, max sequence length)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is a standard Bayesian post-hoc technique applied to reward models."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes the data pipeline: SFT on AlpacaFarm 'sft' split (10k instruction-response pairs), response generation from the SFT policy, gold reward labeling to create proxy training data, and BoN evaluation on 1000 validation prompts with 12,500 responses each."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address the limitations of its synthetic evaluation setup, the small model sizes used, or potential issues with the Laplace approximation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do not show. There is no discussion of which settings (model sizes, datasets, alignment methods beyond BoN) the results may not transfer to."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (generated responses, reward scores, BoN selection results) is made available for independent verification. Only aggregate curves are shown in figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data generation process is described: responses are sampled from the SFT policy, labeled by the gold reward model, and proxy reward models are trained on these synthetic labels (Section 5)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The paper uses synthetic data generated from LLMs and public datasets (AlpacaFarm)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: pretrained Pythia → SFT on AlpacaFarm → generate responses → gold reward labeling → proxy reward model training with LoRA → Laplace-LoRA post-hoc → BoN evaluation on validation prompts (Sections 5 and Appendix C)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed in the paper. There is no acknowledgments section mentioning grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Bristol (authors 1, 7), Huawei Noah's Ark Lab (authors 3, 6), and University College London (authors 4, 5)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Two authors are from Huawei Noah's Ark Lab, which has a commercial interest in LLM alignment methods, but no funding statement is made."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. It trains reward models on synthetic data and evaluates them in a synthetic BoN sampling setup. The evaluation tests the method, not model knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper does not evaluate pre-trained model knowledge on any benchmark. The reward models are trained on synthetic labels and evaluated in a synthetic framework."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — benchmark contamination is not relevant because the paper evaluates a reward modeling method, not model knowledge on existing benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or computational overhead of the Laplace-LoRA approach is reported. The paper does not discuss how much additional computation the Laplace approximation and uncertainty estimation require compared to standard reward models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or hardware specifications are stated. The paper does not quantify the resources needed for training or evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "Laplace-LoRA uncertainty estimates can effectively mitigate reward overoptimization in BoN sampling.",
       "evidence": "Figures 2 and 3 show that Laplace-LoRA reward models (with both variance and std-based penalties) achieve higher gold reward scores at high KL divergence compared to MAP baseline, indicating reduced overoptimization.",
       "supported": "moderate"
     },
     {
       "claim": "Combining Laplace-LoRA with reward ensembles (LA Ens) further enhances performance beyond either approach alone.",
       "evidence": "Figure 3 shows LA Ens outperforms both MAP and standard ensembles (Ens) on gold reward at high KL divergence values.",
       "supported": "moderate"
     },
     {
       "claim": "Variance-based penalty is more robust than standard deviation-based penalty at large k values.",
       "evidence": "Appendix D (Figures 4-5) shows that larger k values (20, 30) degrade performance more for std-based penalty than variance-based penalty.",
       "supported": "moderate"
     },
     {
       "claim": "Laplace-LoRA achieves the highest gold reward in RLHF without KL penalty.",
       "evidence": "Mentioned in the conclusion (Section 7) but no RLHF figures or detailed results are presented in the paper itself.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper proposes using Laplace-LoRA (post-hoc Laplace approximation on LoRA weights) to add uncertainty estimates to reward models for LLM alignment. In a synthetic evaluation using Pythia models and AlpacaFarm data, the uncertainty-penalized reward models mitigate reward overoptimization in best-of-n sampling compared to standard MAP reward models. The approach also combines beneficially with reward model ensembles. However, results are limited to a single synthetic setup with small models (70M proxy, 1.4B policy) and lack statistical rigor (no error bars, significance tests, or multiple runs).",
   "red_flags": [
     {
       "flag": "No error bars or variance across runs",
       "detail": "All results are presented as single curves without any uncertainty quantification. For a paper about uncertainty estimation, the absence of error bars across experimental runs is particularly concerning — the reader cannot assess whether the observed improvements are robust or within noise."
     },
     {
       "flag": "Purely synthetic evaluation",
       "detail": "The gold reward model (LLaMA 7B from AlpacaFarm) serves as both the label generator and ground-truth evaluator. Improvements in gold reward may not translate to actual human preference improvement. No human evaluation is conducted."
     },
     {
       "flag": "Small model sizes",
       "detail": "The proxy reward model is only 70M parameters and the policy is 1.4B. It is unclear whether the Laplace-LoRA approach would be as effective at scales typical of production RLHF systems (7B+ reward models, 70B+ policies)."
     },
     {
       "flag": "RLHF claim without supporting evidence",
       "detail": "The conclusion claims 'This also holds in RLHF, where it achieves the highest gold reward without the application of KL penalty' but no RLHF results are shown in the paper."
     },
     {
       "flag": "No limitations section",
       "detail": "A workshop paper proposing a new method with no discussion of limitations, threats to validity, or scope boundaries."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling laws for reward model overoptimization",
       "authors": ["Leo Gao", "John Schulman", "Jacob Hilton"],
       "year": 2023,
       "relevance": "Foundational work on reward overoptimization that motivates this paper's approach, directly relevant to LLM alignment safety."
     },
     {
       "title": "Reward model ensembles help mitigate overoptimization",
       "authors": ["Thomas Coste", "Usman Anwar", "Robert Kirk", "David Krueger"],
       "year": 2024,
       "relevance": "Key baseline method for mitigating reward hacking through ensembles; this paper extends the approach with Bayesian uncertainty."
     },
     {
       "title": "Helping or herding? Reward model ensembles mitigate but do not eliminate reward hacking",
       "authors": ["Jacob Eisenstein"],
       "year": 2023,
       "arxiv_id": "2312.09244",
       "relevance": "Examines limitations of reward model ensembles for preventing reward hacking in LLM alignment."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang"],
       "year": 2022,
       "relevance": "Foundational RLHF paper describing the reward model + policy optimization paradigm for LLM alignment."
     },
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Yuntao Bai"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "Key work on RLHF for safety alignment, establishing helpful/harmless training objectives."
     },
     {
       "title": "Bayesian low-rank adaptation for large language models",
       "authors": ["Adam X Yang", "Maxime Robeyns", "Xi Wang", "Laurence Aitchison"],
       "year": 2024,
       "relevance": "Introduces Laplace-LoRA, the core technique this paper applies to reward models for uncertainty estimation."
     },
     {
       "title": "Uncertainty-penalized reinforcement learning from human feedback with diverse reward LoRA ensembles",
       "authors": ["Yuanzhao Zhai"],
       "year": 2024,
       "arxiv_id": "2401.00243",
       "relevance": "Proposes diverse LoRA ensembles with uncertainty penalties for RLHF, directly comparable approach to this paper."
     },
     {
       "title": "Improving reinforcement learning from human feedback with efficient reward model ensemble",
       "authors": ["Shun Zhang"],
       "year": 2024,
       "arxiv_id": "2401.16635",
       "relevance": "Explores parameter-efficient reward model ensembles including LoRA ensembles for mitigating reward hacking."
     },
     {
       "title": "WARM: On the benefits of weight averaged reward models",
       "authors": ["Alexandre Ramé"],
       "year": 2024,
       "arxiv_id": "2401.12187",
       "relevance": "Alternative approach to reward model robustness using weight averaging, relevant baseline for reward overoptimization mitigation."
     },
     {
       "title": "ODIN: Disentangled reward mitigates hacking in RLHF",
       "authors": ["Lichang Chen"],
       "year": 2024,
       "arxiv_id": "2402.07319",
       "relevance": "Proposes disentangling reward from response length to mitigate reward hacking, complementary approach to Bayesian uncertainty."
     },
     {
       "title": "AlpacaFarm: A simulation framework for methods that learn from human feedback",
       "authors": ["Yann Dubois"],
       "year": 2024,
       "relevance": "Provides the simulation framework and dataset used for all experiments in this paper."
     }
   ]
 }