ai-research-survey

scan.json (24712B)

---

 {
   "paper": {
     "title": "A Survey on Progress in LLM Alignment from the Perspective of Reward Design",
     "authors": [
       "Miaomiao Ji",
       "Yanqiu Wu",
       "Zhibin Wu",
       "Shoujin Wang",
       "Jian Yang",
       "Mark Dras",
       "Usman Naseem"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2505.02666",
     "doi": "10.48550/arXiv.2505.02666"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This survey taxonomizes reward modeling for LLM alignment along three dimensions: mathematical formulation, construction practices (rule-based, data-driven, hybrid), and functional roles under optimization paradigms (RL-based, ICL-based, DPO/implicit). The paper identifies two major paradigm shifts: from RL-based to RL-free alignment methods, and from single-task to multi-objective/multi-modal reward design. It is a narrative review without systematic methodology or quality assessment of its sources.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository, analysis scripts, or supplementary materials are mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset, paper corpus, or extracted data tables are released. The survey does not provide a downloadable list of reviewed papers or structured extraction results."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided. As a survey, analysis tools or scripts could have been specified but were not."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The survey does not describe a reproducible search or analysis protocol that others could follow."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a narrative survey paper that does not run experiments or report quantitative results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted; no statistical comparisons are made. The paper is a literature review."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted; no effect sizes apply to this narrative survey."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experiments or data collection requiring sample size justification. This is a literature survey."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs are conducted. The paper is a narrative literature review."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper references prior surveys in the Motivation section (refs [26–30]) but does not systematically compare its coverage, methodology, or contributions against them. There is no structured comparison table showing how this survey differs from or improves upon prior reviews."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No experiments requiring baseline comparisons. This is a survey paper."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate. This is a survey paper."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with metrics. This is a survey paper."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate. This is a survey paper."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments requiring test sets. This is a survey paper."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides detailed per-category breakdowns of reward modeling approaches: Table 1 contrasts rule-based, data-driven, and hybrid RMs across multiple dimensions (construction approach, strengths, limitations, typical examples, best-suited scenarios). Table 2 classifies 34 papers across feedback source, optimization paradigm, and RM characteristics."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses failure modes and limitations of different reward design approaches: reward hacking (Section 1.1), reward misspecification (Section 1.2), mode collapse, annotation bias, PPO instability (Section 5.1.1), and limitations of each RM construction paradigm (Table 1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative findings from the literature: PPO's instability and sample inefficiency (Section 5.1.1), reward model overoptimization (ref [22]), hand-engineered metrics conflicting with each other (Section 4.2), and systematic failures of single-value reward modeling (Section 3.1.4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the survey provides 'a structured organization of reward modeling' addressing 'mathematical formulation, construction practices, and interaction with optimization paradigms' with a 'macro-level taxonomy.' Sections 3–5 deliver these three aspects, and Figure 1 provides the taxonomy. The abstract also claims coverage of paradigm shifts from RL-based to RL-free, which Section 6 addresses."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a narrative survey that makes descriptive/taxonomic claims about the field's evolution rather than testable causal claims from its own empirical analysis."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes very broad claims about 'the evolution of LLM alignment' and 'the progression of LLM alignment' without bounding its coverage to specific sub-fields, time periods, or noting what areas of reward design it does not cover. The title and abstract frame the scope broadly without explicit boundaries."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a pure survey/taxonomy paper that presents no empirical results of its own requiring alternative explanations."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no measurements; no proxy-outcome gap can arise from its own methodology."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models are used in this survey. It reviews models discussed in the literature."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. This is a literature survey."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted. This is a literature survey."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a literature survey."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe any paper selection pipeline: no search queries, no databases searched, no inclusion/exclusion criteria, no screening stages. It is unclear how the ~130 cited papers were identified and selected for inclusion."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Section 6 ('Discussion') covers future directions but does not discuss limitations of the survey itself. Section 7 ('Conclusions') is a brief paragraph without self-critical reflection."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential biases in its literature selection, coverage gaps, or limitations of its taxonomic framework."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it does NOT cover. There are no statements about excluded topics, time boundaries for the literature search, or types of reward design approaches that fall outside the review's scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. The complete list of papers reviewed, search results, or extracted classification data is not provided in any downloadable form."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how papers were collected. There is no mention of databases searched, search queries used, date ranges, or how the corpus of reviewed work was assembled."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For a survey, the 'recruitment' of papers requires describing the search and selection strategy. No such description is provided — it is unclear how the reviewed papers were identified and selected from the broader literature."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline is documented. There is no description of stages from initial literature search to final inclusion, no PRISMA-style flow, and no accounting of how many papers were screened vs. included."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section appears in the paper text provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Macquarie University (Australia), Sichuan University (China), and University of Technology Sydney (Australia). None of the authors appear affiliated with companies whose products are being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The absence of a funding statement leaves this unknown."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature survey."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no method of its own to cost."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "The paper follows no PRISMA or structured review protocol. There is no flow diagram, no registered protocol, no reproducible search strategy, and no systematic inclusion/exclusion criteria. It is a narrative review with ad-hoc paper collection."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of its source papers. All cited works are treated equally regardless of their rigor — no quality scoring rubric, risk-of-bias assessment, or evaluation of study design strength is applied."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The survey does not consider whether its sources skew toward positive results, whether negative findings about reward modeling are underrepresented, or whether the reviewed literature has selection effects."
       }
     }
   },
   "claims": [
     {
       "claim": "Reward design is the central connective tissue in the LLM alignment pipeline, bridging feedback collection and optimization.",
       "evidence": "Argued throughout Section 1.2 and Section 2 via a conceptual framework (Figure 2) positioning reward design as the 'prescription' stage between feedback ('diagnosis') and optimization ('treatment').",
       "supported": "moderate"
     },
     {
       "claim": "The evolution of LLM alignment can be understood as a continuous refinement of reward design strategies.",
       "evidence": "Supported narratively through the progression described in Sections 3-6, covering the shift from rule-based to data-driven to hybrid reward modeling, and from explicit to implicit RMs. Table 1 in Section 6 maps 34 papers across these dimensions.",
       "supported": "moderate"
     },
     {
       "claim": "There has been a paradigm shift from RL-based to RL-free optimization methods in LLM alignment.",
       "evidence": "Section 5.3 describes DPO and variants as RL-free alternatives. Section 6.1 asserts this trend. Supported by citations to DPO (ref [70]), SFT-based methods (refs [99-101]), and various DPO variants.",
       "supported": "moderate"
     },
     {
       "claim": "Hybrid reward models that integrate multiple signal sources offer superior robustness and generalization in complex tasks.",
       "evidence": "Section 4.2 reviews hybrid approaches across multi-source, multi-granularity, multi-modal, and multi-aspect dimensions. Individual cited papers report improvements, but the survey does not independently verify or meta-analyze these claims.",
       "supported": "weak"
     },
     {
       "claim": "Token-level reward modeling provides finer credit assignment and more stable training compared to response-level approaches.",
       "evidence": "Section 3.1.3 reviews token-level methods (TLCR ref [34], TDPO ref [36], TLDR ref [37], Q-RM ref [38]) and asserts these benefits, citing individual papers' results without independent verification or meta-analysis.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No systematic review methodology",
       "detail": "The survey follows no PRISMA or structured protocol. There is no documented search strategy, no stated databases, no inclusion/exclusion criteria, and no PRISMA flow diagram. The paper collection appears entirely ad-hoc, making it impossible to assess completeness or selection bias."
     },
     {
       "flag": "No quality assessment of reviewed studies",
       "detail": "All cited papers are treated as equally valid regardless of their methodological rigor. A paper reporting PPO improvements with careful ablation studies is treated identically to a preprint with only a single-run comparison. This launders the signal-to-noise ratio of the sources."
     },
     {
       "flag": "No limitations section",
       "detail": "The survey contains no self-critical discussion of its own methodology, coverage gaps, or potential biases. Section 6 discusses future directions for the field but not limitations of the survey itself."
     },
     {
       "flag": "Broad claims without bounded scope",
       "detail": "The paper claims to capture 'the evolution of LLM alignment' and 'paradigm shifts' without specifying what time period, sub-fields, or venues were covered. No boundaries on what falls outside the review's scope are stated."
     },
     {
       "flag": "Uncritical aggregation of results",
       "detail": "Claims like 'hybrid reward models offer superior robustness' and 'token-level rewards provide more stable training' are stated as general conclusions drawn from individual papers' self-reported improvements, without independent verification or consideration of contradictory evidence."
     }
   ],
   "cited_papers": [
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang"],
       "year": 2022,
       "arxiv_id": "2203.02155",
       "relevance": "InstructGPT — foundational RLHF paper demonstrating instruction-following alignment with human feedback, central to understanding reward modeling pipelines."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell"],
       "year": 2023,
       "arxiv_id": "2305.18290",
       "relevance": "DPO — key RL-free alignment method that implicitly encodes reward through preference optimization, representing a paradigm shift in reward design."
     },
     {
       "title": "Deep reinforcement learning from human preferences",
       "authors": ["Paul F. Christiano", "Jan Leike", "Tom Brown"],
       "year": 2017,
       "relevance": "Foundational RLHF work introducing reward modeling from human preference comparisons, establishing the framework for LLM alignment research."
     },
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Yuntao Bai", "Andy Jones", "Kamal Ndousse"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "Anthropic's RLHF methodology for helpful and harmless AI assistants, demonstrating multi-objective reward design in practice."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI feedback",
       "authors": ["Yuntao Bai", "Saurav Kadavath", "Sandipan Kundu"],
       "year": 2022,
       "arxiv_id": "2212.08073",
       "relevance": "Introduces rule-based constitutional principles combined with AI feedback for alignment, a key hybrid reward design approach."
     },
     {
       "title": "Scaling laws for reward model overoptimization",
       "authors": ["Leo Gao", "John Schulman", "Jacob Hilton"],
       "year": 2023,
       "relevance": "Empirically characterizes reward hacking and overoptimization in RLHF, directly relevant to understanding reward model limitations."
     },
     {
       "title": "Rule based rewards for language model safety",
       "authors": ["Tong Mu", "Alec Helyar", "Johannes Heidecke"],
       "year": 2024,
       "arxiv_id": "2411.01111",
       "relevance": "Proposes rule-based reward framework combining behavioral rules with LLM verification for safety alignment without preference data."
     },
     {
       "title": "DeepSeekMath: Pushing the limits of mathematical reasoning in open language models",
       "authors": ["Zhihong Shao", "Peiyi Wang", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2402.03300",
       "relevance": "Introduces Group Relative Policy Optimization (GRPO), a critic-free RL alternative replacing absolute rewards with group-based relative feedback."
     },
     {
       "title": "Let's verify step by step",
       "authors": ["Hunter Lightman", "Vineet Kosaraju", "Yura Burda"],
       "year": 2023,
       "arxiv_id": "2305.20050",
       "relevance": "Introduces stepwise process rewards for reasoning, demonstrating fine-grained reward design for chain-of-thought alignment."
     },
     {
       "title": "Self-refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan", "Ximing Lu", "Yao Fu"],
       "year": 2023,
       "arxiv_id": "2303.17651",
       "relevance": "Demonstrates non-numerical reward via stepwise self-feedback during generation, an alternative to scalar reward models."
     },
     {
       "title": "Defining and characterizing reward hacking",
       "authors": ["Joar Skalse", "Nikolaus H.R. Howe", "Dmitrii Krasheninnikov"],
       "year": 2022,
       "arxiv_id": "2209.13085",
       "relevance": "Formalizes reward hacking — a core failure mode in reward-based alignment that motivates much of the hybrid reward design research."
     },
     {
       "title": "Safe RLHF: Safe reinforcement learning from human feedback",
       "authors": ["Josef Dai", "Xuehai Pan", "Ruiyang Sun"],
       "year": 2023,
       "arxiv_id": "2310.12773",
       "relevance": "Addresses safety-aware RLHF with constrained reward optimization, relevant to multi-objective reward design for AI safety."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Survey provides conceptual taxonomy useful for researchers but no directly usable tool or technique for practitioners."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms well-known narratives about the RL-to-RL-free shift and the importance of reward design without challenging any conventional wisdom."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Discusses alignment failures, reward hacking, and safety concerns but as a review of known issues rather than revealing new risks."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no criticism of specific labs or methods, purely descriptive taxonomy."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Pure survey paper with no code, demo, or tool to try."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From Macquarie University and Sichuan University — not major AI labs or recognizable brands."
     }
   }
 }