ai-research-survey

scan.json (34133B)

---

 {
   "paper": {
     "title": "MM-RLHF: The Next Step Forward in Multimodal LLM Alignment",
     "authors": [
       "Yi-Fan Zhang",
       "Tao Yu",
       "Haochen Tian",
       "Chaoyou Fu",
       "Peiyan Li",
       "Jianshu Zeng",
       "Wulin Xie",
       "Yang Shi",
       "Huanyu Zhang",
       "Junkang Wu",
       "Xue Wang",
       "Yibo Hu",
       "Bin Wen",
       "Fan Yang",
       "Zhang Zhang",
       "Tingting Gao",
       "Di Zhang",
       "Liang Wang",
       "Rong Jin",
       "Tieniu Tan"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2502.10391",
     "doi": "10.48550/arXiv.2502.10391"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MM-RLHF introduces a 120k human-annotated multimodal preference dataset and a Critique-Based Reward Model that generates critiques before scoring, achieving SOTA among open-source 7B reward models and surpassing several 72B models. Combined with Dynamic Reward Scaling (MM-DPO), alignment training on LLaVA-OV-7B yields substantial gains in conversational benchmarks (~19.5%) and safety (~60% reduction in unsafe behavior) while also improving hallucination, math reasoning, and video understanding. The paper also finds that self-improvement of small-scale MLLMs (<7B) is currently unrealistic due to model capacity constraints and limitations in existing reward signal quality.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a project page URL (https://mm-rlhf.github.io/) but no explicit source code repository link (e.g., GitHub). No working code URL is provided in the paper text."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the MM-RLHF dataset (120k pairs) in detail and provides a project page URL, but does not include an explicit dataset download link or statement that the dataset is publicly available for download."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '32×H800 (80G) GPUs' for hardware but provides no requirements.txt, Dockerfile, conda environment, or detailed library version listing."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README commands, or replication scripts are provided in the paper or appendix."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2, 3, 4, and 5 report only point estimates. No confidence intervals, error bars, or ± notation is present for any experimental result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparative claims (e.g., '19.5% increase', 'surpassing several 72B models') based solely on comparing numbers without any statistical significance tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports both baseline and aligned model scores with absolute differences (e.g., '+7.20', '+4.60'), providing sufficient context to assess magnitude. Table 3 similarly shows before/after differences for safety metrics."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for key sample size decisions: why 30k queries were sampled from 10M, why the 4:5:1 sampling ratio was chosen (beyond 'diversity goals'), or why 120k pairs are sufficient for alignment."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measure is reported across experimental runs. Results appear to be single-run numbers with no indication of result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 compares aligned models against their unaligned baselines (LLaVA-OV-7B, LLaVA-OV-0.5B, InternVL2-1B). Table 4 compares the reward model against LLaVA-OV-7B, LLaVA-Critic, and GPT-4o. Table 5 compares against 10+ open and closed-source models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4o (2024-08-06), Claude-3.5-Sonnet (2024-06-22), Gemini-1.5-Pro (2024-09-24), Qwen2-VL-72B, InternVL2-26B — all contemporary state-of-the-art models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 12(a) ablates MM-RLHF dataset, implicit reward, and MM-DPO components. Table 4 ablates reward model components (w/o Task 1, w/o enhanced annotations). Figure 12(b) studies sensitivity to hyperparameters w and k."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates across 27 benchmarks spanning 10 dimensions: conversation, general knowledge, chart/document, OCR, real-world, math, hallucination, video, multi-image, and safety."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All system evaluation is automated (standard benchmarks + GPT-4o as judge). The human annotation is used for dataset creation, not for evaluating the aligned model's outputs. Section 5.1 states 'For all benchmarks requiring GPT-assisted evaluation, we consistently employ GPT-4o as the evaluation model.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses established benchmarks (POPE, MME, MathVista, etc.) with standard test splits. MM-RLHF-RewardBench is separately sampled from the dataset (Section 3.2: 'we randomly sample 10 examples from each category'). Training data is distinct from evaluation benchmarks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-benchmark results across 8 evaluation dimensions. Table 3 provides per-task safety breakdowns. Table 4 provides per-category reward model accuracy (MCQ, Long, Short, Safety, Video)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses 'Limited gains in high-resolution benchmarks' with specific analysis of why. Section 5.4 discusses why self-improvement fails for small MLLMs. Appendix D shows failure examples from annotation (Figures 8-11)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results: high-resolution benchmarks show no improvement or decreases (Table 2, e.g., MME-RealWorld cn -0.62%); self-improvement is 'currently unrealistic' (Section 5.4); multiple sampling of critiques 'does not yield significant performance gains' (Section 5.3); implicit reward strategy doesn't work for MLLMs (Appendix E)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claim of '19.5% increase in conversational abilities' for LLaVA-OV-7B is supported by Table 2's conversation metrics showing substantial gains. The '60% improvement in safety' aligns with Table 3 showing unsafety rate dropping from 40.2% to 13.9% for LLaVA-OV-7B. The body's average figures (11%, 57%) refer to averages across all models, which is a different but consistent scope."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('alignment training leads to improvements', 'Dynamic Reward Scaling enhances MM-DPO') supported by ablation studies: Figure 12(a) shows incremental contributions of dataset, reward model, and DPO algorithm via controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'The Next Step Forward in Multimodal LLM Alignment' implies broad generality, but experiments cover only 3 base models (LLaVA-OV-7B, LLaVA-OV-0.5B, InternVL2-1B), all ≤7B parameters. No explicit bounding to these specific model families or scales. The paper does not discuss whether results generalize to larger models or different architectures."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No systematic consideration of alternative explanations for the main results. Could improvements stem from the additional SFT loss component? From exposure to more diverse data during alignment rather than the alignment algorithm itself? Section 5.2 notes model-specific preferences but doesn't discuss confounds."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures benchmark scores and frames them as 'alignment with human preferences', 'conversational abilities', and 'safety improvement.' No discussion of whether benchmark performance on automated metrics (many GPT-4o-judged) is a faithful proxy for actual human preference alignment. The gap between automated benchmark scores and genuine human satisfaction is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 specifies model versions with dates: 'GPT-4o (2024-08-06)', 'Claude-3.5-Sonnet (2024-06-22)', 'GPT-4o-mini (2024-07-18)', 'Gemini-1.5-Pro (2024-09-24)'. Open-source models are specified by name and size (e.g., 'Qwen2-VL-72B-Instruct', 'LLaVA-OV-7B')."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Table 7 shows one prompt (for augmenting human annotations). Appendix B provides annotation guidelines. However, the prompts used for model response generation, GPT-4o evaluation, and safety data construction are not provided. The reader cannot reconstruct the full prompting setup."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 reports: SFT loss weight search over {0, 0.1, 0.25, 0.5, 1.0}, learning rate search over {1e-7, 5e-7, 1e-6, 5e-6, 1e-5}, βori = 0.1, default w = 0.5, k = 0.5. Vision encoder frozen. Maximum 8 samples for self-improvement experiments."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. It focuses on dataset construction, reward modeling, and alignment training, none of which involve agent-based workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 documents data filtering: predefined sampling weights across 3 categories (MCQ 4.14%, Long 12.17%, Short 83.68% → adjusted ratio 4:5:1), CLIP-based KNN clustering with 100 cluster centers, and deduplication strategy. Table 1 shows final composition. Figure 1 illustrates the full pipeline."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. Section 6 ('Conclusion and Future Work') briefly mentions underutilized annotation granularity and high-resolution data limitations, but this is primarily forward-looking rather than a substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Several study-specific threats are discussed, though scattered: 'our dataset contains relatively few ultra-high-resolution images' (Section 5.2), 'our filtering strategy is based on image similarity rather than resolution' (Section 5.2), 'the cost of human annotation poses scalability challenges' (Section 2.3.2), and the finding that implicit reward doesn't transfer from LLMs to MLLMs (Appendix E)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no systematic discussion of excluded populations/settings, untested model scales, or claims the authors are NOT making. The broad title suggests general applicability without explicit bounding."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The project page (https://mm-rlhf.github.io/) is referenced but no explicit raw data download link is provided in the paper. The 120k annotated pairs, raw annotations, and scoring details are not confirmed as publicly downloadable."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes data sources (LLaVA-OV, VLfeedback, LLaVA-RLHF, lrv-instruction, Unimm-Chat, SharedGPT-4 video, VLGuard). Section 2.2 describes filtering. Section 2.3 describes annotation procedures including timing ('average of over 8 minutes' per question) and quality control."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Section 2.3.2 states 'over 50 annotators, supported by 8 multimodal research experts with strong English proficiency and academic backgrounds' but does not describe how these annotators were recruited — via what platform, institution, or selection process. The recruitment channel and potential selection bias are not discussed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Figure 1 and Sections 2.1-2.3: 10M instruction samples → clustering/deduplication/sampling → ~30k queries → response generation with 4 SOTA models → human annotation → 120k comparison pairs. Table 1 provides final composition counts. Appendix C details safety data construction."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments section is present in the paper. The note mentions 'Work done during an internship at KuaiShou Group' but no grants or funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: KuaiShou, CASIA, NJU, USTC, PKU, Alibaba, Meta AI. The note states 'Work done during an internship at KuaiShou Group.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "KuaiShou (a Chinese tech company that develops short-video platforms and apps) has a potential commercial interest in improved MLLM alignment for its products. The work was done at KuaiShou (4 authors affiliated), and no statement of funder independence is provided."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Authors from KuaiShou, Alibaba, and Meta AI may have commercial interests related to the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for the base models (LLaVA-OV-7B, InternVL2-1B) or for the models used in response generation (GPT-4o, Claude 3.5 Sonnet, Qwen2-VL-72B). This is necessary to assess whether benchmark examples could have been in pre-training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether benchmark test examples (POPE, MME, MathVista, etc.) appeared in the pre-training data of the evaluated models. No overlap analysis or decontamination is mentioned."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Many benchmarks used (POPE 2023, MME 2023, TextVQA 2019, VQAv2 2015) predate the models' likely training cutoffs. No discussion of contamination risk for these established benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper has no human participants as research subjects. Human annotators are workers creating the dataset, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects research is conducted. Annotators create training data but are not research participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The annotators are described as having 'strong English proficiency and academic backgrounds' but this is workforce description, not participant demographics."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API costs, tokens consumed, or per-example latency is reported for the alignment training, reward model, or the aligned models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '32×H800 (80G) GPUs' as hardware but does not state GPU hours, wall-clock training time, or total compute budget. The annotation cost ('over 8 minutes per question' × 30k queries) is partially described but total cost is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. All results appear to be from single runs with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never explicitly stated. Results are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 specifies the search space: grid search over SFT loss weight {0, 0.1, 0.25, 0.5, 1.0} and learning rate {1e-7, 5e-7, 1e-6, 5e-6, 1e-5}. Figure 12(b) shows the w × k search grid. The method (grid search) and space (25 configurations for main search) are clear."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions selecting the 'best-performing configuration' from the grid search but does not specify which metric or validation set was used for selection. The final hyperparameter choices are not justified beyond 'default values of w = 0.5 and k = 0.5 work well.'"
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparisons across 27 benchmarks, 10 dimensions, and multiple models without any correction for multiple comparisons (no Bonferroni, Holm, or similar corrections)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own MM-RLHF dataset, MM-RLHF-Reward model, and MM-DPO algorithm without acknowledging self-evaluation bias. Their reward model is used to score pairs that then train their own alignment method."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance-vs-compute analysis is provided. The paper does not discuss whether the improvements justify the compute cost, or compare methods at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 27 benchmarks without questioning whether they measure what is claimed. For example, GPT-4o-judged conversation benchmarks may not reflect actual human preference alignment, but this validity gap is not discussed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding comparisons are made. The paper compares alignment methods applied to the same base models, not different scaffolds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Many benchmarks (VQAv2 from 2015, POPE from 2023, etc.) predate the models' training, but the paper does not address whether solutions appeared in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For instance, whether the alignment training data overlaps with any evaluation benchmarks is not analyzed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test data. The MM-RLHF training data sources overlap with some evaluation benchmark sources (both draw from LLaVA and similar multimodal datasets), but this is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied (no canary strings, membership inference tests, n-gram overlap analysis, or decontamination pipelines)."
       }
     }
   },
   "claims": [
     {
       "claim": "Fine-tuning LLaVA-OV-7B with MM-RLHF leads to a 19.5% increase in conversational abilities and a 60% improvement in safety.",
       "evidence": "Table 2 shows substantial conversation improvements (e.g., WildVision win rate +22.0, LLaVA-Wild detail +12.20). Table 3 shows unsafety rate dropping from 40.2% to 13.9% for LLaVA-OV-7B. However, the exact '19.5%' calculation methodology is not specified.",
       "supported": "moderate"
     },
     {
       "claim": "MM-RLHF-Reward-7B achieves SOTA performance among open-source reward models, surpassing several 72B-scale models.",
       "evidence": "Table 5 shows MM-RLHF-Reward-7B (50.15 avg) outperforming Qwen2-VL-72B-Instruct (42.97), NVLM-D-72B (44.17), and InternVL2-26B (45.67) on VLRewardBench. Table 4 shows 85% ACC on MM-RLHF-RewardBench.",
       "supported": "strong"
     },
     {
       "claim": "Critique-based training significantly enhances reward model quality: ACC+ improves from 50% to 67%.",
       "evidence": "Table 4 shows progressive improvement: w/o Task 1 (50% ACC+) → w/o enhanced annotations (57% ACC+) → full model (67% ACC+) → with GT annotations (87% ACC+). Controlled ablation on the same dataset and base model.",
       "supported": "strong"
     },
     {
       "claim": "Dynamic Reward Scaling (MM-DPO) further improves alignment performance beyond standard DPO.",
       "evidence": "Figure 12(a) shows average score increasing from 74.9 (MM-RLHF + standard DPO) to 76.5 (MM-DPO) on the 4-benchmark evaluation. However, ablation uses 1/5 of data, and the improvement margin is modest.",
       "supported": "moderate"
     },
     {
       "claim": "Self-improvement of small-scale MLLMs (<7B parameters) is currently unrealistic for comprehensive performance gains.",
       "evidence": "Figure 6 shows self-sampled data with the best available reward model (73.9 avg) significantly underperforms human-annotated data (75.0 avg). Section 5.4 discusses model capacity constraints and reward signal quality limitations.",
       "supported": "moderate"
     },
     {
       "claim": "Alignment training improves multi-image and video understanding despite no dedicated multi-image data in the training set.",
       "evidence": "Table 2 shows improvements for LLaVA-OV-7B: MMMU-Pro +1.33%, LLAVA-Next-Interleave +0.27%, VideoChatGPT +0.35. Improvements are modest and lack error bars to confirm they exceed noise.",
       "supported": "weak"
     },
     {
       "claim": "Existing multimodal reward models (e.g., LLaVA-Critic) exhibit significant overfitting to their training data domains.",
       "evidence": "Table 4 shows LLaVA-Critic achieving only 45% ACC (pointwise) and 35% ACC (pairwise) vs. GPT-4o's 74% on MM-RLHF-RewardBench. Section 5.3 attributes this to training data limited to 'conversational datasets and real-world images.'",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results across 27 benchmarks, 3 models, and multiple ablations are point estimates. No variance, standard deviation, confidence intervals, or significance tests are reported, making it impossible to assess whether observed differences are statistically meaningful."
     },
     {
       "flag": "Self-evaluation bias",
       "detail": "The authors evaluate their own dataset (MM-RLHF), reward model (MM-RLHF-Reward), alignment algorithm (MM-DPO), and benchmarks (MM-RLHF-RewardBench, MM-RLHF-SafetyBench). The reward model used to score training pairs for MM-DPO is trained on the same dataset. No independent evaluation is conducted."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Many evaluation benchmarks (VQAv2, POPE, TextVQA, etc.) substantially predate the models used. No training cutoff dates are stated for any model, and no contamination analysis is performed despite evaluating on 27 established benchmarks."
     },
     {
       "flag": "Ablation on reduced data",
       "detail": "Section 5.1 states ablation studies use 'uniformly sample 1/5 of the data, which may result in minor performance discrepancies compared to the full dataset.' This weakens the ablation conclusions as they may not reflect the full-data setting."
     },
     {
       "flag": "Circular evaluation methodology",
       "detail": "MM-RLHF-Reward is trained on MM-RLHF data, then used to provide reward signals for MM-DPO training on MM-RLHF data. MM-RLHF-RewardBench is sampled from MM-RLHF data. This circular dependency could inflate performance metrics — the system is partially evaluated on data it was designed around."
     },
     {
       "flag": "Undisclosed conflicts of interest",
       "detail": "Four authors are from KuaiShou (a large Chinese tech company), and the work was done during an internship there. No funding disclosure, competing interests statement, or acknowledgment of commercial interest in improved MLLM alignment is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Aligning large multimodal models with factually augmented rlhf",
       "authors": ["Zhiqing Sun", "Sheng Shen", "Shengcao Cao", "Haotian Liu"],
       "year": 2023,
       "arxiv_id": "2309.14525",
       "relevance": "First multimodal RLHF algorithm (Fact-RLHF), pioneering work on aligning MLLMs with human feedback that this paper directly builds upon."
     },
     {
       "title": "LLaVA-Critic: Learning to Evaluate Multimodal Models",
       "authors": ["Tianyi Xiong", "Xiyao Wang", "Dong Guo"],
       "year": 2024,
       "arxiv_id": "2410.02712",
       "relevance": "Key baseline for MLLM self-evaluation and reward modeling, demonstrating iterative DPO strategy for MLLM alignment."
     },
     {
       "title": "RLHF-V: Towards Trustworthy MLLMs via Behavior Alignment from Fine-grained Correctional Human Feedback",
       "authors": ["Tianyu Yu", "Yuan Yao", "Haoye Zhang"],
       "year": 2024,
       "relevance": "Prior work on fine-grained human feedback for MLLM alignment, addressing hallucination through behavioral alignment."
     },
     {
       "title": "RLAIF-V: Aligning MLLMs through Open-Source AI Feedback for Super GPT-4V Trustworthiness",
       "authors": ["Tianyu Yu", "Haoye Zhang", "Yuan Yao"],
       "year": 2024,
       "arxiv_id": "2405.17220",
       "relevance": "Explores AI feedback as an alternative to human feedback for MLLM alignment, directly relevant to the human vs. machine annotation debate."
     },
     {
       "title": "Silkie: Preference Distillation for Large Visual Language Models",
       "authors": ["Lei Li", "Zhihui Xie", "Mukai Li"],
       "year": 2023,
       "arxiv_id": "2312.10665",
       "relevance": "VLfeedback dataset used as a data source in MM-RLHF construction; explores AI-generated preference data for VLM alignment."
     },
     {
       "title": "Direct Preference Optimization with an Offset",
       "authors": ["Afra Amini", "Tim Vieira", "Ryan Cotterell"],
       "year": 2024,
       "arxiv_id": "2402.10571",
       "relevance": "DPO variant that adjusts optimization based on sample quality — related to MM-DPO's Dynamic Reward Scaling approach."
     },
     {
       "title": "Beta-DPO: Direct Preference Optimization with Dynamic Beta",
       "authors": ["Junkang Wu", "Yuexiang Xie", "Zhengyi Yang"],
       "year": 2024,
       "arxiv_id": "2407.08639",
       "relevance": "Prior work on dynamic beta adjustment in DPO for LLMs, which the paper shows does not directly transfer to MLLMs."
     },
     {
       "title": "Self-generated critiques boost reward modeling for language models",
       "authors": ["Yue Yu", "Zhengxing Chen", "Aston Zhang"],
       "year": 2024,
       "arxiv_id": "2411.16646",
       "relevance": "Explores critique-based reward models in LLMs; this paper extends the concept to multimodal settings with distinct design choices."
     },
     {
       "title": "Safety fine-tuning at (almost) no cost: A baseline for vision large language models",
       "authors": ["Yongshuo Zong", "Ondrej Bohdal", "Tingyang Yu"],
       "year": 2024,
       "arxiv_id": "2402.02207",
       "relevance": "VLGuard dataset used for safety data construction in MM-RLHF; establishes safety fine-tuning baselines for VLMs."
     },
     {
       "title": "VLRewardBench: A Challenging Benchmark for Vision-Language Generative Reward Models",
       "authors": ["Lei Li", "Yuancheng Wei", "Zhihui Xie"],
       "year": 2024,
       "arxiv_id": "2411.17451",
       "relevance": "Benchmark for evaluating multimodal reward models, used as one of the evaluation settings for MM-RLHF-Reward."
     },
     {
       "title": "Provably Robust DPO: Aligning Language Models with Noisy Feedback",
       "authors": ["Sayak Ray Chowdhury", "Anush Kini", "Nagarajan Natarajan"],
       "year": 2024,
       "arxiv_id": "2403.00409",
       "relevance": "Addresses robustness of DPO to noisy preference data, related to MM-DPO's approach of weighting samples by quality."
     },
     {
       "title": "Red Teaming Visual Language Models",
       "authors": ["Mukai Li", "Lei Li", "Yuwei Yin"],
       "year": 2024,
       "arxiv_id": "2401.12915",
       "relevance": "Red teaming approach for VLMs, relevant to safety evaluation and adversarial robustness of multimodal models."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides a concrete dataset and training recipe for MLLM alignment that practitioners could adopt, though implementation requires significant compute resources."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Challenges the view that alignment only helps specific tasks and shows self-improvement doesn't work for small MLLMs, but the main results (RLHF helps) are largely expected."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Includes a safety alignment component and safety benchmark (jailbreaking, adversarial attacks), but the focus is on improving safety rather than demonstrating novel attacks."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle; presents straightforward methodology improvement results."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Project page exists (https://mm-rlhf.github.io/) but no pip-installable tool or live demo is available; reproducing requires substantial infrastructure."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from KuaiShou, CASIA, and Meta AI have some recognition but are not among the most prominent Western AI labs; MM-RLHF itself is new."
     }
   }
 }