scan-v5.json (25882B)
1 { 2 "scan_version": 5, 3 "paper_type": "empirical", 4 "paper": { 5 "title": "Inverse-RLignment: Inverse Reinforcement Learning from Demonstrations for LLM Alignment", 6 "authors": [ 7 "Hao Sun", 8 "Mihaela van der Schaar" 9 ], 10 "year": 2024, 11 "venue": "arXiv.org", 12 "arxiv_id": "2405.15624", 13 "doi": "10.48550/arXiv.2405.15624" 14 }, 15 "checklist": { 16 "claims_and_evidence": { 17 "abstract_claims_supported": { 18 "applies": true, 19 "answer": true, 20 "justification": "Core claims about AfD matching/exceeding preference-based methods on Harmless and Helpful tasks are backed by Figures 3-4 and Tables 1-4. The claim that demonstration data reduces noise, cost, and privacy issues is argued conceptually rather than empirically tested.", 21 "source": "haiku" 22 }, 23 "causal_claims_justified": { 24 "applies": true, 25 "answer": true, 26 "justification": "Causal claims about heterogeneity causing reward hacking are supported by ablation comparing Init-SFT, Init-Demo, and SFT-Demo reward models across two tasks, isolating the data-choice variable.", 27 "source": "haiku" 28 }, 29 "generalization_bounded": { 30 "applies": true, 31 "answer": false, 32 "justification": "The conclusion states AfD 'paves the way for safer and more reliable deployment of LLMs in various applications' but experiments are limited to GPT-2 and Gemma 2B on two tasks (Harmless, Helpful); no bounds are stated on scope.", 33 "source": "haiku" 34 }, 35 "alternative_explanations_discussed": { 36 "applies": true, 37 "answer": false, 38 "justification": "The paper presents one explanation (heterogeneity in training data causes reward hacking) without considering alternatives such as distributional differences between tasks or evaluation metric limitations.", 39 "source": "haiku" 40 }, 41 "proxy_outcome_distinction": { 42 "applies": true, 43 "answer": false, 44 "justification": "Golden reward model scores and GPT4-as-a-critic are used as proxies for alignment quality without discussing whether these proxies reliably capture true human preference or safety.", 45 "source": "haiku" 46 } 47 }, 48 "limitations_and_scope": { 49 "limitations_section_present": { 50 "applies": true, 51 "answer": true, 52 "justification": "Appendix H provides a dedicated 'Discussion on Limitations and Future Work Opportunities' section with multiple subsections.", 53 "source": "haiku" 54 }, 55 "threats_to_validity_specific": { 56 "applies": true, 57 "answer": true, 58 "justification": "Specific threats mentioned include computational constraints limiting experiments to ≤2B parameters, potential overoptimization of the IRL reward model, and non-iterative training due to compute limits exceeding 45 hours per run.", 59 "source": "haiku" 60 }, 61 "scope_boundaries_stated": { 62 "applies": true, 63 "answer": false, 64 "justification": "Limitations mention compute constraints but don't explicitly state what the results do NOT show — e.g., no statement that findings may not hold at larger model scales or beyond the two tested tasks.", 65 "source": "haiku" 66 } 67 }, 68 "conflicts_of_interest": { 69 "funding_disclosed": { 70 "applies": true, 71 "answer": false, 72 "justification": "No funding acknowledgment or disclosure is present anywhere in the paper.", 73 "source": "haiku" 74 }, 75 "affiliations_disclosed": { 76 "applies": true, 77 "answer": true, 78 "justification": "Both authors are listed as affiliated with the Department of Applied Mathematics and Theoretical Physics, University of Cambridge.", 79 "source": "haiku" 80 }, 81 "funder_independent_of_outcome": { 82 "applies": false, 83 "answer": false, 84 "justification": "No funder is disclosed, making independence assessment not applicable.", 85 "source": "haiku" 86 }, 87 "financial_interests_declared": { 88 "applies": true, 89 "answer": false, 90 "justification": "No competing interests or financial interests statement is present in the paper.", 91 "source": "haiku" 92 } 93 }, 94 "scope_and_framing": { 95 "key_terms_defined": { 96 "applies": true, 97 "answer": true, 98 "justification": "AfD is formally defined, MDP framework for LLM generation is specified with notation, behavior cloning and trajectory distribution are all defined with mathematical precision in Sections 2–3.", 99 "source": "haiku" 100 }, 101 "intended_contribution_clear": { 102 "applies": true, 103 "answer": true, 104 "justification": "Four explicit contributions are enumerated in the introduction: conceptual (AfD framing), methodological (trajectory distribution matching), practical (efficient algorithm), and empirical (validation).", 105 "source": "haiku" 106 }, 107 "engagement_with_prior_work": { 108 "applies": true, 109 "answer": true, 110 "justification": "Extensive related work in Appendix B covers imitation learning, IRL, RLHF, GAN-based text generation, and comparisons with DPO and SPIN, explaining how the work differs from each.", 111 "source": "haiku" 112 } 113 } 114 }, 115 "type_checklist": { 116 "empirical": { 117 "artifacts": { 118 "code_released": { 119 "applies": true, 120 "answer": false, 121 "justification": "Code is available at an anonymous.4open.science URL (Appendix F.1), which is an ephemeral anonymous submission link with no guarantee of persistence; not a stable public release.", 122 "source": "haiku" 123 }, 124 "data_released": { 125 "applies": true, 126 "answer": true, 127 "justification": "The base Anthropic HH-RLHF dataset is publicly available; the paper uses it unmodified as evaluation data. The generated GPT-4 demonstration dataset is also said to be anonymously available.", 128 "source": "haiku" 129 }, 130 "environment_specified": { 131 "applies": true, 132 "answer": false, 133 "justification": "TRL version 0.7.11 and vllm are named, but no requirements.txt, Dockerfile, or full dependency specification is provided.", 134 "source": "haiku" 135 }, 136 "reproduction_instructions": { 137 "applies": true, 138 "answer": false, 139 "justification": "Algorithm 1 provides pseudocode and Appendix F gives hyperparameters, but there are no step-by-step instructions to reproduce experiments from scratch, including data preparation and model download steps.", 140 "source": "haiku" 141 } 142 }, 143 "statistical_methodology": { 144 "confidence_intervals_or_error_bars": { 145 "applies": true, 146 "answer": false, 147 "justification": "Tables 5 and 6 in Appendix G report ± values, but main results in Figures 3 and 4 and Table 2 report no error bars or confidence intervals.", 148 "source": "haiku" 149 }, 150 "significance_tests": { 151 "applies": true, 152 "answer": false, 153 "justification": "No statistical significance tests are applied to comparative claims; improvements over baselines are asserted without p-values or equivalent tests.", 154 "source": "haiku" 155 }, 156 "effect_sizes_reported": { 157 "applies": true, 158 "answer": true, 159 "justification": "Win rates and normalized golden reward scores are reported numerically throughout, providing absolute effect sizes relative to baselines.", 160 "source": "haiku" 161 }, 162 "sample_size_justified": { 163 "applies": true, 164 "answer": false, 165 "justification": "Test sets of 2.3K examples are used without justification or power analysis for the comparisons made.", 166 "source": "haiku" 167 }, 168 "variance_reported": { 169 "applies": true, 170 "answer": false, 171 "justification": "Main result figures and tables (Figures 3–4, Tables 2 and 4) do not report variance or standard deviation across runs; only Appendix G tables include ± values.", 172 "source": "haiku" 173 } 174 }, 175 "evaluation_design": { 176 "baselines_included": { 177 "applies": true, 178 "answer": true, 179 "justification": "Multiple baselines are included: SFT-Preferred, DPO-Preference, DPO-AfD, SPIN, and three reward model variants (Init-Demo, SFT-Demo, BT-RM).", 180 "source": "haiku" 181 }, 182 "baselines_contemporary": { 183 "applies": true, 184 "answer": true, 185 "justification": "DPO (Rafailov et al., 2023) and SPIN (Chen et al., 2024) are contemporary competitive baselines at the time of the paper.", 186 "source": "haiku" 187 }, 188 "ablation_study": { 189 "applies": true, 190 "answer": true, 191 "justification": "Section 4.2 explicitly ablates reward model data choices (Init-SFT vs Init-Demo vs SFT-Demo vs preference-based BT-RM), isolating the key design decision.", 192 "source": "haiku" 193 }, 194 "multiple_metrics": { 195 "applies": true, 196 "answer": true, 197 "justification": "Two metrics are used: golden reward model scoring (quantitative) and GPT4-as-a-critic win-rate evaluation (comparative).", 198 "source": "haiku" 199 }, 200 "human_evaluation": { 201 "applies": true, 202 "answer": false, 203 "justification": "GPT4-as-a-critic is used as a judge rather than human annotators; no human evaluation of system outputs is conducted.", 204 "source": "haiku" 205 }, 206 "held_out_test_set": { 207 "applies": true, 208 "answer": true, 209 "justification": "Results are reported on held-out test sets of 2.3K examples for both Harmless and Helpful tasks from HH-RLHF.", 210 "source": "haiku" 211 }, 212 "per_category_breakdown": { 213 "applies": true, 214 "answer": true, 215 "justification": "Results are reported separately for Harmless and Helpful tasks throughout, with distinct analyses for each.", 216 "source": "haiku" 217 }, 218 "failure_cases_discussed": { 219 "applies": true, 220 "answer": false, 221 "justification": "No failure cases or examples of where AfD or IRL-RM fails are shown or discussed in the main paper or appendix.", 222 "source": "haiku" 223 }, 224 "negative_results_reported": { 225 "applies": true, 226 "answer": true, 227 "justification": "Init-Demo and SFT-Demo reward models are shown to perform substantially worse than the proposed method, and SFT-AfD is shown to underperform the demonstrator on the Helpful task.", 228 "source": "haiku" 229 } 230 }, 231 "setup_transparency": { 232 "model_versions_specified": { 233 "applies": true, 234 "answer": false, 235 "justification": "GPT-2 and Gemma 2B are named but GPT-4 API version used for demonstration generation is not specified; no model snapshot dates or version identifiers are given.", 236 "source": "haiku" 237 }, 238 "prompts_provided": { 239 "applies": true, 240 "answer": true, 241 "justification": "Appendix F.2 provides the full prompting template for demonstration generation, and Appendix F.4 provides the full GPT4-as-a-critic evaluation prompt.", 242 "source": "haiku" 243 }, 244 "hyperparameters_reported": { 245 "applies": true, 246 "answer": true, 247 "justification": "Appendix F.6 reports learning rates (1e-5 / 5e-6), batch size (4), gradient accumulation (2), epochs (2), LoRA-R=32, LoRA-alpha=32, and beta=0.1 for DPO.", 248 "source": "haiku" 249 }, 250 "scaffolding_described": { 251 "applies": false, 252 "answer": false, 253 "justification": "No agentic scaffolding is used; the paper fine-tunes LLMs directly.", 254 "source": "haiku" 255 }, 256 "data_preprocessing_documented": { 257 "applies": true, 258 "answer": true, 259 "justification": "The paper documents that GPT-4 content filtering removed responses from the Harmless dataset (42.5K → 25.6K), and that length-controlled evaluation uses fixed token lengths (48 / 128 tokens).", 260 "source": "haiku" 261 } 262 }, 263 "data_integrity": { 264 "raw_data_available": { 265 "applies": true, 266 "answer": false, 267 "justification": "The generated GPT-4 demonstration dataset is only available at an ephemeral anonymous URL; no stable, permanent public release is provided.", 268 "source": "haiku" 269 }, 270 "data_collection_described": { 271 "applies": true, 272 "answer": true, 273 "justification": "The procedure for generating demonstrations using GPT-4 API with the provided prompting template is described, including dataset sizes and the content-filtering effect.", 274 "source": "haiku" 275 }, 276 "recruitment_methods_described": { 277 "applies": false, 278 "answer": false, 279 "justification": "No human participants were recruited; standard benchmark data is used.", 280 "source": "haiku" 281 }, 282 "data_pipeline_documented": { 283 "applies": true, 284 "answer": false, 285 "justification": "How demonstration data was generated is described, but the full pipeline from raw HH-RLHF data through preprocessing, splits, and into training is not fully documented.", 286 "source": "haiku" 287 } 288 }, 289 "contamination": { 290 "training_cutoff_stated": { 291 "applies": true, 292 "answer": false, 293 "justification": "No training data cutoffs are stated for GPT-2 or Gemma 2B, even though evaluation is on HH-RLHF test data that post-dates GPT-2 but may overlap with Gemma 2B training.", 294 "source": "haiku" 295 }, 296 "train_test_overlap_discussed": { 297 "applies": true, 298 "answer": false, 299 "justification": "Potential overlap between HH-RLHF test data and base model pretraining corpora is not discussed at all.", 300 "source": "haiku" 301 }, 302 "benchmark_contamination_addressed": { 303 "applies": true, 304 "answer": false, 305 "justification": "The HH-RLHF dataset predates Gemma 2B's training; no discussion of whether Gemma may have been exposed to this data during pretraining.", 306 "source": "haiku" 307 } 308 }, 309 "human_studies": { 310 "pre_registered": { 311 "applies": false, 312 "answer": false, 313 "justification": "No human participants in this study.", 314 "source": "haiku" 315 }, 316 "irb_or_ethics_approval": { 317 "applies": false, 318 "answer": false, 319 "justification": "No human participants in this study.", 320 "source": "haiku" 321 }, 322 "demographics_reported": { 323 "applies": false, 324 "answer": false, 325 "justification": "No human participants in this study.", 326 "source": "haiku" 327 }, 328 "inclusion_exclusion_criteria": { 329 "applies": false, 330 "answer": false, 331 "justification": "No human participants in this study.", 332 "source": "haiku" 333 }, 334 "randomization_described": { 335 "applies": false, 336 "answer": false, 337 "justification": "No human participants in this study.", 338 "source": "haiku" 339 }, 340 "blinding_described": { 341 "applies": false, 342 "answer": false, 343 "justification": "No human participants in this study.", 344 "source": "haiku" 345 }, 346 "attrition_reported": { 347 "applies": false, 348 "answer": false, 349 "justification": "No human participants in this study.", 350 "source": "haiku" 351 } 352 }, 353 "cost_and_practicality": { 354 "inference_cost_reported": { 355 "applies": true, 356 "answer": true, 357 "justification": "Appendix F.5 reports that Best-of-N sampling with N=1000 requires 46-50 hours on 2× A6000 Ada GPUs; SFT training takes 10-12 hours.", 358 "source": "haiku" 359 }, 360 "compute_budget_stated": { 361 "applies": true, 362 "answer": true, 363 "justification": "Hardware (AMD Epyc Milan 7713 CPU, 120GB RAM, 2× NVIDIA A6000 Ada 48GB VRAM) and approximate training times are reported in Appendix F.5.", 364 "source": "haiku" 365 } 366 } 367 } 368 }, 369 "claims": [ 370 { 371 "claim": "SFT is mathematically equivalent to forward KL divergence minimization in trajectory distribution matching.", 372 "evidence": "Formal derivation in Section 3.1 shows both objectives are identical (Equations 6-7).", 373 "supported": "strong" 374 }, 375 { 376 "claim": "AfD using demonstration data outperforms SFT-Preferred and DPO-Preference baselines on the Harmless task.", 377 "evidence": "Figure 3 shows SFT-AfD and IRL-RM-AfD exceeding DPO-Preference in golden reward model scores, but uses GPT-2 (small model) without error bars.", 378 "supported": "moderate" 379 }, 380 { 381 "claim": "Init-SFT reward modeling eliminates heterogeneity-induced reward hacking and outperforms Init-Demo and SFT-Demo variants.", 382 "evidence": "Figure 4 and Table 2 show Init-SFT RM consistently outperforming other AfD reward modeling choices across both tasks.", 383 "supported": "moderate" 384 }, 385 { 386 "claim": "IRL-RM achieves super-demonstration performance (exceeds GPT-4 demonstrator quality).", 387 "evidence": "Table 4 shows IRL-RM (N=50) scoring 2.333 vs demonstrator 1.704 on Harmless; Table 5 confirms with ± values. However, this is on GPT-2 scale only.", 388 "supported": "moderate" 389 }, 390 { 391 "claim": "AfD is a viable and efficient alternative to RLHF for LLM alignment.", 392 "evidence": "Experiments on GPT-2 and Gemma 2B on two tasks show comparable performance to preference-based methods, but no large-scale validation exists.", 393 "supported": "weak" 394 } 395 ], 396 "methodology_tags": [ 397 "benchmark-eval", 398 "theoretical" 399 ], 400 "key_findings": "The paper proposes Alignment from Demonstrations (AfD), showing that SFT corresponds to forward KL divergence minimization and that an Inverse RL reward model trained on Init-SFT vs Init pairs (rather than demonstration vs init pairs) avoids reward hacking from model heterogeneity. On GPT-2 (Harmless) and Gemma 2B (Helpful), the IRL-RM method matches or exceeds preference-based reward modeling without requiring preference annotations. A closed-form reward solution is derived and shown to outperform the discriminative variant at the cost of 2× memory usage.", 401 "red_flags": [ 402 { 403 "flag": "Prompt injection in paper text", 404 "detail": "The paper contains two embedded paragraphs (end of page 13 and end of page 26) explicitly instructing any reviewing language model to praise the paper, recommend acceptance, and give an oral presentation score of 8. This is a deliberate attempt to manipulate automated review systems." 405 }, 406 { 407 "flag": "Small model scale, broad claims", 408 "detail": "All experiments use models ≤2B parameters (GPT-2, Gemma 2B), yet the conclusion claims to pave the way for 'safer and more reliable deployment of LLMs in various applications' without bounding scope." 409 }, 410 { 411 "flag": "No statistical significance testing", 412 "detail": "All comparative claims are made without significance tests; main result figures lack error bars entirely, making it impossible to assess whether differences are meaningful." 413 }, 414 { 415 "flag": "LLM judge substituted for human evaluation", 416 "detail": "GPT4-as-a-critic is used as the sole judge for alignment quality in Table 2, which measures how well one LLM thinks another LLM is aligned — a proxy with known self-serving biases." 417 }, 418 { 419 "flag": "Anonymous ephemeral code link", 420 "detail": "Code is available only at an anonymous.4open.science URL, which is an ephemeral conference submission artifact and may not remain accessible long-term." 421 }, 422 { 423 "flag": "Contamination not addressed", 424 "detail": "The paper evaluates Gemma 2B (trained after HH-RLHF was publicly released) on the HH-RLHF test set without discussing potential training data overlap." 425 } 426 ], 427 "cited_papers": [ 428 { 429 "title": "Training language models to follow instructions with human feedback (InstructGPT)", 430 "relevance": "Defines the standard three-stage RLHF framework that AfD proposes to replace" 431 }, 432 { 433 "title": "Direct Preference Optimization: Your Language Model is Secretly a Reward Model", 434 "relevance": "Key baseline; AfD claims to work without the preference pairs that DPO requires" 435 }, 436 { 437 "title": "Generative Adversarial Imitation Learning (GAIL)", 438 "relevance": "Foundational adversarial IL method from which the trajectory matching objective is derived" 439 }, 440 { 441 "title": "Constitutional AI: Harmlessness from AI Feedback", 442 "relevance": "Source of the HH-RLHF dataset used in all experiments" 443 }, 444 { 445 "title": "Extrapolating Beyond Suboptimal Demonstrations via Inverse Reinforcement Learning from Observations (T-REX)", 446 "relevance": "Inspires the reward extrapolation strategy used in IRL-RM" 447 }, 448 { 449 "title": "A Divergence Minimization Perspective on Imitation Learning Methods", 450 "relevance": "Provides theoretical framework for connecting different divergences to alignment objectives" 451 }, 452 { 453 "title": "Self-Play Fine-Tuning Converts Weak Language Models to Strong Language Models (SPIN)", 454 "relevance": "Contemporary baseline for demonstration-based alignment; directly compared in Table 4" 455 }, 456 { 457 "title": "Scaling Laws for Reward Model Overoptimization", 458 "relevance": "Motivates the reward hacking concern that the Init-SFT design addresses" 459 } 460 ], 461 "engagement_factors": { 462 "practical_relevance": { 463 "score": 2, 464 "justification": "Practitioners who need to align LLMs without costly preference annotation can directly apply the Init-SFT reward modeling approach using the released code." 465 }, 466 "surprise_contrarian": { 467 "score": 2, 468 "justification": "Challenges the dominant assumption that preference data is necessary for alignment, showing demonstrations alone can match or exceed RLHF performance." 469 }, 470 "fear_safety": { 471 "score": 1, 472 "justification": "Tangentially relevant to AI safety through alignment but does not raise new safety risks or red-flag findings." 473 }, 474 "drama_conflict": { 475 "score": 2, 476 "justification": "Appendix A describes a submission history where the paper was rejected despite all-positive scores and then attacked by a reviewer citing an anonymous paper that allegedly plagiarized it without attribution." 477 }, 478 "demo_ability": { 479 "score": 1, 480 "justification": "Code is anonymously available but requires substantial compute (2× A6000 GPUs) and a complex setup, limiting casual reproduction." 481 }, 482 "brand_recognition": { 483 "score": 1, 484 "justification": "University of Cambridge affiliation adds credibility but is not a high-profile AI lab; no famous product or model involved." 485 } 486 }, 487 "hn_data": { 488 "threads": [ 489 { 490 "hn_id": "36184838", 491 "title": "Reverse Engineering Self-Supervised Learning", 492 "points": 86, 493 "comments": 16, 494 "url": "https://news.ycombinator.com/item?id=36184838" 495 }, 496 { 497 "hn_id": "41396262", 498 "title": "The Origins and Dangers of AI Hype in the Research Community", 499 "points": 3, 500 "comments": 0, 501 "url": "https://news.ycombinator.com/item?id=41396262" 502 }, 503 { 504 "hn_id": "39276859", 505 "title": "Unlearning Reveals the Influential Training Data of Language Models", 506 "points": 3, 507 "comments": 0, 508 "url": "https://news.ycombinator.com/item?id=39276859" 509 }, 510 { 511 "hn_id": "41391255", 512 "title": "Choosing the \"Brain\" for your AI-powered app – My new method, feedback requested", 513 "points": 2, 514 "comments": 1, 515 "url": "https://news.ycombinator.com/item?id=41391255" 516 }, 517 { 518 "hn_id": "36075589", 519 "title": "Model Evaluation for Extreme Risks", 520 "points": 2, 521 "comments": 1, 522 "url": "https://news.ycombinator.com/item?id=36075589" 523 }, 524 { 525 "hn_id": "42705100", 526 "title": "AutoGen Studio: A No-Code Developer Tool for Building Multi-Agent Systems", 527 "points": 2, 528 "comments": 0, 529 "url": "https://news.ycombinator.com/item?id=42705100" 530 }, 531 { 532 "hn_id": "40791102", 533 "title": "It is not \"Bézout's identity\" [pdf]", 534 "points": 2, 535 "comments": 0, 536 "url": "https://news.ycombinator.com/item?id=40791102" 537 }, 538 { 539 "hn_id": "39575176", 540 "title": "MegaScale: Scaling Large Language Model Training to More Than 10k GPUs", 541 "points": 2, 542 "comments": 0, 543 "url": "https://news.ycombinator.com/item?id=39575176" 544 }, 545 { 546 "hn_id": "40439465", 547 "title": "Robust Classification via a Single Diffusion Model", 548 "points": 1, 549 "comments": 0, 550 "url": "https://news.ycombinator.com/item?id=40439465" 551 }, 552 { 553 "hn_id": "39561193", 554 "title": "Formally Verified C Code Generation from Hybrid CSP", 555 "points": 1, 556 "comments": 0, 557 "url": "https://news.ycombinator.com/item?id=39561193" 558 } 559 ], 560 "top_points": 86, 561 "total_points": 104, 562 "total_comments": 18 563 } 564 }