ai-research-survey

scan.json (30195B)

---

 {
   "paper": {
     "title": "Getting More Juice Out of the SFT Data: Reward Learning from Human Demonstration Improves SFT for LLM Alignment",
     "authors": [
       "Jiaxiang Li",
       "Siliang Zeng",
       "Hoi-To Wai",
       "Chenliang Li",
       "Alfredo Garcia",
       "Mingyi Hong"
     ],
     "year": 2024,
     "venue": "Neural Information Processing Systems",
     "arxiv_id": "2405.17888",
     "doi": "10.48550/arXiv.2405.17888"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "The paper proposes two algorithms (RFT and IRFT) that learn reward models from demonstration data during SFT via inverse reinforcement learning. IRFT improves zephyr-7b-sft-full from 59.48% to 61.03% on the HuggingFace Open LLM Leaderboard. The implicit reward learned from demonstration data alone can distinguish preferred from non-preferred continuations without seeing preference data. IRFT generalizes SPIN as a special case (T=1) and is shown to converge to stationary points with O(1/√TK) rate.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code is released at https://github.com/JasonJiaxiangLi/Reward_learning_SFT, stated in the abstract and confirmed in the NeurIPS checklist as released upon acceptance."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used are publicly available: Anthropic-HH (Bai et al., 2022) and Ultrachat200k from HuggingFace. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C specifies: DeepSpeed ZeRO-3, FlashAttention-2, RMSProp optimizer, NVIDIA A100-40G GPUs (2 for 1B, 8 for 7B), bfloat16 precision, TRL package for PPO, and Language Model Evaluation Harness v0.4.2."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released and Appendix C provides hyperparameters, the paper does not contain explicit step-by-step reproduction instructions (e.g., specific commands to run). The paper says 'We follow the code as in SPIN' but does not provide a self-contained reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Figure 2 shows error bands for the reward/win-rate plots, but the main benchmark results in Tables 3, 4, and 6 report only point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Claims like 'the proposed methods show significant performance improvement' are based on raw number comparisons (e.g., 59.48% vs 61.03%) without any p-values or tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance is reported with baseline context across all experiments. Tables 3 and 4 show absolute scores for each method and task, allowing computation of improvement magnitudes (e.g., 59.48% → 61.03% = 1.55pp improvement)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the choice of 10k data from Anthropic-HH or 50k from Ultrachat200k. No power analysis. The 50k selection follows SPIN's convention but is not independently justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Figure 2 shows variance bands for reward/win-rate curves, but the main benchmark results in Tables 3, 4, and 6 report single-run numbers with no standard deviations or variance across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include the pretrained model, standard SFT, and SPIN (Chen et al., 2024) across all experiments in Tables 3, 4, and 6."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "SPIN (Chen et al., 2024) is a contemporary baseline from the same year. DPO is discussed conceptually. The baselines represent recent approaches to SFT improvement."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3, 4, and 6 systematically vary the outer loop count T and inner loop count K, effectively ablating the key hyperparameters of the proposed algorithms. Both RFT and IRFT variants are tested."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Six metrics from the Open LLM Leaderboard are used: AI2_Arc, TruthfulQA, Winogrande, GSM8k, HellaSwag, and MMLU. Additionally, reward scores and win rates are reported for Anthropic-HH experiments."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated via the Open LLM Leaderboard benchmarks and a reward model (PKU-Alignment/beaver-7b-v3.0-reward). No human evaluation of model outputs was conducted, though Table 5 shows qualitative generation examples without human rating."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Open LLM Leaderboard uses standard test splits. For Anthropic-HH, they evaluate on a test dataset separate from the 10k training set (Figure 2 caption: 'average score of test dataset')."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3, 4, and 6 provide per-task breakdowns across all six Open LLM Leaderboard tasks, not just averages."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated failure analysis. The paper notes '1b model is not strong enough to handle hard tasks, e.g. GSM8k' but does not analyze where or why the proposed methods fail. Table 5 shows only successful generation examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 shows IRFT T=8 (59.85%) and T=16 (60.25%) underperforming other IRFT configurations. The paper notes 'more frequent generation might also result in more variances' and acknowledges that hyperparameter tuning is needed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'significant performance improvement over existing SFT approaches' and cites 59.47%→61.03%. Table 4 confirms: base zephyr-7b-sft-full at 59.48% and IRFT T=10 at 61.03%. Convergence guarantees are proven in Theorem 3.1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('reward learning improves SFT') are supported by controlled experiments varying only the training method while holding model, data, and hyperparameters fixed. Ablation over T and K values provides additional causal evidence through single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Reward Learning from Human Demonstration Improves SFT for LLM Alignment' broadly, but experiments are limited to pythia-1.4b and zephyr-7b-sft-full on two datasets. The conclusion acknowledges 'future works include exploring reward-learning for larger models' but the broad framing is not adequately bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the observed improvements. For example, the improvement could be due to the additional training compute (generation + extra training steps) rather than reward learning per se. The SPIN comparison partially addresses this but not explicitly."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures Open LLM Leaderboard scores (commonsense reasoning, math, knowledge) and frames this as 'alignment' improvement. The gap between benchmark performance and actual alignment quality is not acknowledged — benchmark scores are treated as direct measures of alignment."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names are provided: pythia-1b, pythia-1.4b, zephyr-7b-sft-full, PKU-Alignment/beaver-7b-v3.0-reward. These are HuggingFace model identifiers that pinpoint exact model versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C states: 'We use the same prompt template \"### Instruction: prompt\\n\\n### Response:\" as in Chen et al. [2024].' The template is fully specified and input content comes from public datasets."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C reports: learning rate 5e-7 (first 2 epochs) / 1e-7 (next 2), β=0.1, max sequence length 1024 (1B) / 2048 (7B), per-device batch sizes of 4/8/2, RMSProp optimizer, bfloat16 precision, 2 epochs per generation."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The methods are training algorithms (SFT, reward learning, policy optimization) without any agentic workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes: for Anthropic-HH, only chosen/preferred continuations are kept; top 10k data selected by beaver-7b-v3.0-reward scores. For Ultrachat200k, same 50k selection strategy as SPIN. Training for 2 epochs per generation round."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Conclusions and Limitations' discusses: convergence only to stationary points (not global optima), unclear what the policy converges to, and non-negligible additional computation costs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations discuss theoretical scope (stationary point convergence) and computational cost, but do not address specific threats to validity of the empirical results, such as whether improvements are within noise margins, confounding from additional compute, or model-specific effects."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'future works include exploring reward-learning for larger models and more complicated demonstration tasks' but does not explicitly state what the current results do NOT show, such as which settings or model scales the findings should not be generalized to."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All experimental data comes from publicly available datasets: Anthropic-HH and Ultrachat200k from HuggingFace. Code for reproducing experiments is released on GitHub."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes data collection: Anthropic-HH provides preference data from which only chosen continuations are used; 10k selected by reward scores. Ultrachat200k is a subset of UltraChat with 50k selected for training."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks (Anthropic-HH, Ultrachat200k, Open LLM Leaderboard tasks)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: raw datasets → selection of chosen/preferred continuations → filtering by reward score (for Anthropic-HH) or following SPIN's 50k strategy → training with 2 epochs per generation → evaluation on test splits."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section: 'M. Hong, S. Zeng and J. Li are supported partially by NSF under the grants EPCN-2311007, ECCS-2426064 and CCF-1910385, also by Minnesota Supercomputing Institute. A. Garcia and C. Li are partially supported by ECCS-2240789.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All six authors list university affiliations: University of Minnesota, Chinese University of Hong Kong, and Texas A&M University. No evaluated product is affiliated with these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from NSF grants and Minnesota Supercomputing Institute. NSF is an independent federal agency with no financial interest in the outcome of LLM alignment research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff for pythia-1.4b and zephyr-7b-sft-full is not stated. These models are evaluated on Open LLM Leaderboard benchmarks that may have been in their pre-training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Open LLM Leaderboard benchmark data (Arc, HellaSwag, MMLU, etc.) appeared in the pre-training data of pythia or zephyr models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses benchmarks (Arc 2018, HellaSwag 2019, MMLU 2020, Winogrande 2021) that were all published before the models' training data was collected, creating contamination risk. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments involve model training and automated benchmark evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Study involves only LLM training algorithms and automated evaluation."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 provides qualitative cost comparisons (e.g., 'SFT+Generation' time) but no actual wall-clock times, GPU hours, or dollar costs are reported for any experiment."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (2× or 8× NVIDIA A100-40G) but total GPU hours, training time, or total compute budget is not stated for any experiment."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Tables 3, 4, and 6 appear to report single-run results without seed variation analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. Results appear to be from single runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Different T and K values are tested (Tables 3, 4, 6) but no formal hyperparameter search budget is reported. The search space and total configurations tried are not quantified."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3, 4, and 6 report all tested configurations (various T and K combinations) rather than cherry-picking the best result. The paper recommends T=5 as a default based on the observed pattern across experiments."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across methods and tasks without any significance tests, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own baselines (SFT, SPIN) without acknowledging potential author-implementation bias. No independent evaluation or discussion of this bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 provides qualitative cost categories but performance is not plotted as a function of compute. IRFT uses more compute than SFT (generation overhead) but this is not controlled for in comparisons."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Open LLM Leaderboard benchmarks to measure 'alignment' quality without discussing whether these benchmarks actually measure alignment. Commonsense reasoning and math tasks are proxies for alignment at best."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The methods are training algorithms evaluated directly through standard benchmark pipelines."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark tasks (Arc 2018, MMLU 2020, etc.) existed before the models' pre-training data was collected."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides information that would not be available in real deployment."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and evaluation data. The SFT data (Anthropic-HH, Ultrachat) and evaluation benchmarks may share distributional overlap."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied (no canary strings, membership inference, decontamination, or temporal splits)."
       }
     }
   },
   "claims": [
     {
       "claim": "Reward learning from demonstration data significantly improves SFT for LLM alignment, with IRFT improving zephyr-7b-sft-full from 59.48% to 61.03% average on the Open LLM Leaderboard.",
       "evidence": "Table 4 shows zephyr-7b-sft-full base at 59.48% average, IRFT T=10 at 61.03%. Multiple IRFT configurations outperform both SFT and the base model.",
       "supported": "moderate"
     },
     {
       "claim": "IRL-based methods can distinguish preferred from non-preferred continuations even when trained only on preferred/chosen demonstration data.",
       "evidence": "Figure 1 (right) shows log probability gap between chosen and rejected continuations. RFT and IRFT produce positive gaps while SFT assigns higher probability to non-preferred data. Example 2 in Section 3.",
       "supported": "moderate"
     },
     {
       "claim": "The proposed algorithms (RFT and IRFT) converge to stationary solutions of the IRL problem at rate O(1/√TK).",
       "evidence": "Theorem 3.1 with formal proof in Appendix B under Assumption B.1 (bounded policy, bounded gradients, Lipschitz gradients).",
       "supported": "strong"
     },
     {
       "claim": "IRFT generalizes SPIN as a special case: IRFT with T=1 is equivalent to SPIN.",
       "evidence": "Section 4 shows mathematically that when T=1 and K equals total training iterations, IRFT reduces to SPIN. Confirmed empirically in Tables 3 and 4 where IRFT T=1 rows match SPIN.",
       "supported": "strong"
     },
     {
       "claim": "More frequent generation (T>1) in IRFT outperforms the single-generation SPIN approach, with T≈5-10 being optimal.",
       "evidence": "Tables 3 and 4 show IRFT with various T values. For 7B: SPIN (T=1) 60.32%, IRFT T=5 at 60.71%, T=10 at 61.03%. But T=8 at 59.85% is worse, suggesting sensitivity.",
       "supported": "moderate"
     },
     {
       "claim": "The reward model learned from only demonstration data (via RFT) already possesses strong capability in distinguishing chosen and rejected responses.",
       "evidence": "Figure 1 (right) and Figure 2 showing win rate of RFT over SFT models reaching ~0.8. The reward is learned without any preference data.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars on main benchmark results",
       "detail": "Tables 3, 4, and 6 report single-run point estimates. Differences as small as 0.08% (e.g., 37.27 vs 37.35 in Table 3) are presented as meaningful improvements without any uncertainty quantification. These could easily be within noise."
     },
     {
       "flag": "No significance tests for claimed improvements",
       "detail": "The paper claims 'significant performance improvement' based solely on comparing raw numbers. No statistical tests are used to establish that 59.48%→61.03% is statistically significant rather than random variation."
     },
     {
       "flag": "Inconsistent hyperparameter sensitivity",
       "detail": "IRFT T=8 on 7B model (59.85%) actually underperforms IRFT T=1/SPIN (60.32%), while T=10 is the best (61.03%). This non-monotonic pattern and sensitivity to T suggests results may be fragile and the 'optimal' T may be coincidental."
     },
     {
       "flag": "Compute confound not controlled",
       "detail": "IRFT with T>1 involves more generation passes than SFT or SPIN, meaning more total compute. The improvement could be partially or fully explained by additional training compute rather than the reward learning mechanism itself. Table 2 acknowledges the cost difference but does not control for it experimentally."
     },
     {
       "flag": "Benchmark-alignment proxy gap unacknowledged",
       "detail": "Open LLM Leaderboard tasks (commonsense reasoning, math, knowledge) are used as proxies for 'alignment quality' without discussion of whether these benchmarks actually measure the alignment improvements claimed."
     }
   ],
   "cited_papers": [
     {
       "title": "Deep reinforcement learning from human preferences",
       "authors": ["Paul F Christiano", "Jan Leike", "Tom Brown", "Miljan Martic", "Shane Legg", "Dario Amodei"],
       "year": 2017,
       "relevance": "Foundational RLHF paper that established the reward-learning-then-RL pipeline for aligning models with human preferences."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell", "Christopher D Manning", "Stefano Ermon", "Chelsea Finn"],
       "year": 2024,
       "relevance": "DPO is a key baseline approach for alignment that bypasses explicit reward modeling; this paper builds on DPO's implicit reward identity."
     },
     {
       "title": "Self-play fine-tuning converts weak language models to strong language models",
       "authors": ["Zixiang Chen", "Yihe Deng", "Huizhuo Yuan", "Kaixuan Ji", "Quanquan Gu"],
       "year": 2024,
       "arxiv_id": "2401.01335",
       "relevance": "SPIN is a primary baseline and the paper shows IRFT generalizes SPIN; both use demonstration data with self-generated synthetic negatives."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "InstructGPT paper that established the SFT→RLHF pipeline used in ChatGPT, which this work seeks to improve at the SFT stage."
     },
     {
       "title": "Zephyr: Direct distillation of lm alignment",
       "authors": ["Lewis Tunstall", "Edward Beeching", "Nathan Lambert"],
       "year": 2023,
       "arxiv_id": "2310.16944",
       "relevance": "Provides the zephyr-7b-sft-full base model used in experiments and represents the DPO alignment approach."
     },
     {
       "title": "Is DPO superior to PPO for LLM alignment? A comprehensive study",
       "authors": ["Shusheng Xu", "Wei Fu", "Jiaxuan Gao"],
       "year": 2024,
       "arxiv_id": "2404.10719",
       "relevance": "Comprehensive comparison of reward-based vs reward-free LLM alignment methods, directly relevant to the reward learning debate."
     },
     {
       "title": "Proximal policy optimization algorithms",
       "authors": ["John Schulman", "Filip Wolski", "Prafulla Dhariwal", "Alec Radford", "Oleg Klimov"],
       "year": 2017,
       "arxiv_id": "1707.06347",
       "relevance": "PPO is used for the policy optimization step in Algorithm 1 (RFT) and is the standard RL algorithm in RLHF pipelines."
     },
     {
       "title": "Self-rewarding language models",
       "authors": ["Weizhe Yuan", "Richard Yuanzhe Pang", "Kyunghyun Cho"],
       "year": 2024,
       "arxiv_id": "2401.10020",
       "relevance": "Related approach using self-generated rewards for LLM improvement, part of the broader trend of reward learning without explicit preference data."
     },
     {
       "title": "Safe RLHF: Safe reinforcement learning from human feedback",
       "authors": ["Josef Dai", "Xuehai Pan", "Ruiyang Sun", "Jiaming Ji"],
       "year": 2024,
       "relevance": "Provides the beaver-7b-v3.0-reward model used as the ground-truth reward evaluator in the Anthropic-HH experiments."
     },
     {
       "title": "Back to basics: Revisiting reinforce style optimization for learning from human feedback in LLMs",
       "authors": ["Arash Ahmadian", "Chris Cremer", "Matthias Gallé"],
       "year": 2024,
       "arxiv_id": "2402.14740",
       "relevance": "Revisits REINFORCE-based policy optimization for RLHF, part of the ongoing exploration of RL methods for LLM alignment."
     },
     {
       "title": "Weak-to-strong generalization: Eliciting strong capabilities with weak supervision",
       "authors": ["Collin Burns", "Pavel Izmailov", "Jan Hendrik Kirchner"],
       "year": 2023,
       "arxiv_id": "2312.09390",
       "relevance": "Explores using weaker supervision signals to improve stronger models, related to the question of extracting more signal from limited demonstration data."
     },
     {
       "title": "Maximum-likelihood inverse reinforcement learning with finite-time guarantees",
       "authors": ["Siliang Zeng", "Chenliang Li", "Alfredo Garcia", "Mingyi Hong"],
       "year": 2022,
       "relevance": "Foundational IRL work by the same authors that provides the ML-IRL framework used in this paper's formulation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners doing SFT alignment could apply IRFT as a drop-in improvement; code is released and the method uses standard training infrastructure."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Challenges the assumption that reward learning requires preference data, but IRL from demonstrations is well-established in the RL literature."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the work is about improving SFT training efficiency."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy; the paper takes a constructive approach building on existing methods (SPIN, DPO, RLHF)."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub with implementation following SPIN's codebase, making it runnable by researchers with appropriate GPU resources."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "University research groups (Minnesota, CUHK, Texas A&M) without major brand recognition in the LLM space."
     }
   }
 }