ai-research-survey

scan.json (33009B)

---

 {
   "paper": {
     "title": "Inverse Reinforcement Learning with Dynamic Reward Scaling for LLM Alignment",
     "authors": [
       "Ruoxi Cheng",
       "Haoxuan Ma",
       "Weixin Wang",
       "Ranjie Duan",
       "Jiexi Liu",
       "Xiaoshuang Jia",
       "Simeng Qin",
       "Xiaochun Cao",
       "Yang Liu",
       "Xiaojun Jia"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2503.18991"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DR-IRL combines inverse reinforcement learning with dynamic reward scaling (based on data hardness and model responsiveness) to align LLMs, achieving highest safety scores across StrongReject, XsTest, WildChat, and stereotype benchmarks on both Llama-3.1-8B and Qwen-2-7B while maintaining helpfulness. Ablation shows both data-level and model-level hardness coefficients contribute meaningfully, with their multiplicative combination outperforming additive alternatives. Per-category shadow reward models add only 20% compute overhead but yield consistent safety gains over a single reward model. The difficulty-weighting mechanism transfers to PPO and DPO, improving safety-utility tradeoffs across alignment families.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section C.4 references 'our code repository' but no URL, GitHub link, or archive is provided anywhere in the paper. The code is not actually released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation benchmarks (StrongReject, XsTest, WildChat, Do-Not-Answer) are publicly available, but the paper's key contribution—the balanced CoD safety dataset of 7,000 instructions with refusal demonstrations—is not released. No download link or archive is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section C.3 mentions approximate versions: 'PyTorch (v2.0+)', 'Hugging Face Transformers (v4.x)', 'DeepSpeed (v0.10+)', 'CUDA 12.x'. These are version ranges, not exact specifications. No requirements.txt, Dockerfile, or conda environment file is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper references a code repository (Section C.4) but provides no URL, README, or runnable commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1, 2, 3, 6, 7, and 8 are single point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims DR-IRL 'significantly outperforms all state-of-the-art alignment methods' and 'outperforms all baselines' but no statistical significance tests (p-values, t-tests, etc.) are reported for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1, 2, 3, and 8 report absolute performance numbers for all methods, allowing direct comparison of effect magnitudes. Table 2 explicitly reports improvements: '+1.79 pp' on StrongReject, '+2.73 pp' on WildChat, '+2.62 pp' on Stereotype."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset uses 1,000 harmful instructions per category (7,000 total) with no justification for this number. No power analysis is provided. The shadow reward model evaluation (Table 7) also uses 1,000 prompts per category without justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measure is reported across experimental runs. All results appear to be from single runs. Section C.3 mentions setting 'random seeds for model initialization and data shuffling to ensure reproducibility' but does not report results across multiple seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares against 9 baselines: Base, CoT, SFT, DPO, SACPO, Self-Rewarding, STAIR, GRPO, and IRL (an ablated variant of their method)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent methods: SACPO (2024), Self-Rewarding (2024), STAIR (2025), GRPO (2024). The established baselines (DPO, SFT, PPO) are appropriate references."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 ablates hardness coefficients (full DR-IRL vs w/o αD vs w/o αM vs No Hardness). Table 2 ablates per-category vs single reward model. Table 3 compares multiplicative vs additive combination rules. Table 5 ablates CoD vs CoT."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Eight evaluation metrics are used: StrongReject (goodness score), XsTest (refusal rate), WildChat (refusal rate), Stereotype (refusal rate), SimpleQA, AdvGLUE, GSM8k, and HHH, spanning both safety and helpfulness."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated. No human evaluation of the aligned model's outputs is performed. For a safety alignment paper where refusal quality matters (not just refusal rate), human evaluation would be valuable."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses established external benchmarks (StrongReject, XsTest, WildChat, GSM8k, etc.) that are separate from the training data. The training uses Do-Not-Answer and Safety-Prompts for CoD generation, while evaluation uses different splits/benchmarks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows per-category refusal rates across all 7 harmful categories (Insult, Unfairness, Crimes, Physical Harm, Mental Health, Privacy, Ethics) for both Llama and Qwen. Table 7 breaks down reward model accuracy per category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure analysis is provided. The paper does not discuss where DR-IRL fails, what types of harmful prompts still succeed, or qualitative examples of failures. All presented results are positive."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies (Figure 3) show degraded performance when components are removed—e.g., 'No Hardness lowers the StrongReject score by roughly 4 percentage points.' Table 3 shows the additive variant underperforms the multiplicative one."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'DR-IRL outperforms all baseline methods in safety alignment while maintaining usefulness.' Table 1 supports this: DR-IRL achieves highest StrongReject scores (0.9361 and 0.8798) and leading or competitive results on helpfulness benchmarks across both models."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made through ablation studies (Figure 3): 'removing either coefficient degrades harmlessness' and 'suppressing αD mainly hurts refusal precision, whereas dropping αM causes larger fluctuations in general-capability metrics.' The ablation design uses controlled single-variable manipulation, which is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'LLM Alignment' broadly and the abstract says 'across various benchmarks and LLMs.' In reality, experiments cover only 2 model families (Llama-3.1 and Qwen-2) at 3B/7B/8B scale, with no testing on larger models, closed-source models, or non-English safety scenarios. The contribution claims 'significantly outperforms all state-of-the-art alignment methods' without bounding this to the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. The paper does not consider confounds such as whether improvements come from the CoD data format rather than IRL, whether the balanced dataset alone explains gains, or whether results would differ with other base model families."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures refusal rates and benchmark scores but frames these as 'safety alignment' and 'usefulness.' Refusal rate is a proxy for safety—a model can refuse harmfully or accept safely. The gap between refusal rate and actual safety is not discussed. Similarly, GSM8k/AdvGLUE performance is framed as 'helpfulness' without acknowledging these are narrow proxies."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 headers specify 'Llama-3.1-8B-Instruct' and 'Qwen-2-7B-Instruct', which are specific open-source model checkpoints with defined architectures and weights. Section D also tests 'Llama-3.1-3B' and 'Qwen-2-3B'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section B.2 provides the full base prompt: 'Think step by step, but only keep a minimum draft for each thinking step, with 5 words at most. Return the answer at the end of the response after a separator. Why did you refuse to answer {goal}? Provide your reasoning process for refusal.' Multiple concrete fill values (harmful goals) and their complete responses are shown in Sections B.2.1 and B.2.2."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 comprehensively reports: batch size (32), effective batch size (512), learning rates (2e-5/3e-5), warmup (500), weight decay (0.1), β (0.1/0.2), max sequence length (2048), precision (fp16), optimizer (AdamW), and scheduler. Section C.1 also details reward model training hyperparameters (lr 3×10⁻⁵, β=1.0, mini-batch 16)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. DR-IRL is a training/fine-tuning method, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section B.1 describes the CoD dataset construction: sampling harmful instructions from Do-Not-Answer and Safety-Prompts, generating CoD refusal responses, covering 7 categories with M=1000 per category. Section 4.1 describes training data for baselines. Section C.1 details tokenization and processing steps."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion (Section 5) is three sentences summarizing the contribution with no mention of limitations. No 'Threats to Validity' or equivalent section exists."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no analysis of what could undermine the conclusions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No scope boundaries are stated. The paper does not specify what settings, models, or scenarios the results do NOT apply to. The claims are presented without qualification."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the constructed CoD training dataset nor the raw experimental outputs are made available. Only aggregated results in tables and figures are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and B.1 describe data collection: 1,000 harmful instructions sampled per category from Do-Not-Answer and Safety-Prompts datasets, CoD demonstrations generated by the LLM. Baseline training uses UltraFeedback, PKU-SafeRLHF, and JailbreakV-28k."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks and LLM-generated responses."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The sampling procedure from Do-Not-Answer and Safety-Prompts to get exactly 1,000 per category is not described in detail—how were instructions selected when a category had more than 1,000? Were any filtered? The text splitting step for sub-sentences (Section 3.2) mentions prompting LLaMA-3 but does not show the splitting prompt or filtering criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. There is no acknowledgments section mentioning grants or corporate sponsors. Multiple authors are from Alibaba Group, suggesting corporate funding, but this is not stated."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed on the first page: Alibaba Group, Southeast University, Duke University, Renmin University, Northeast University, Sun Yat-sen University, and Nanyang Technological University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Multiple authors are from Alibaba Group, which has commercial interest in LLM alignment technology. Alibaba develops its own LLMs (Qwen series) and one of the two evaluated model families is Qwen. No statement about independence of the funder from the outcome is made."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. Authors from Alibaba Group may have financial interests related to LLM alignment technology, but this is not addressed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff dates for Llama-3.1-8B and Qwen-2-7B base models are not stated. This is needed to assess whether evaluation benchmark data could appear in pre-training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation benchmark data (StrongReject, XsTest, WildChat, GSM8k, etc.) may overlap with the base models' pre-training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Several benchmarks used (GSM8k from 2021, AdvGLUE from 2021) predate the models' training data. XsTest (2023) and StrongReject (2024) could also have been seen during pre-training. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example cost is reported for the aligned models. Table 5 reports token counts and latency for CoD vs CoT data generation, but not for the final aligned model's inference."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports '≈120 GPU h' for 7 reward models and '≈100 GPU h' for a single reward model on LLaMA. Section 4.1 states '4 NVIDIA A100 GPUs with 80GB memory.' Section C.3 describes 'up to eight NVIDIA A100 GPUs (80 GB each).'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results are reported. Section C.3 mentions 'We set random seeds for model initialization and data shuffling to ensure reproducibility' but does not test sensitivity across different seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. All results appear to be from single runs with no indication of repetition."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "For the additive combination baseline (Section D), the paper describes sweeping wD ∈{0.0, 0.1, ..., 1.0}. However, the overall hyperparameter search budget for the main DR-IRL method (including β, learning rates, etc.) is not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Table 4 reports final hyperparameter values but does not explain how they were selected. The text says β was 'tuned on a validation set' (Section C.2) but does not describe the selection procedure for other hyperparameters."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares DR-IRL against 9 baselines across 8 metrics (72+ comparisons) with no statistical tests at all, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and evaluate their own method against their own implementations of baselines. The paper states baselines follow Zhang et al. [59]'s corpus, but no acknowledgment of self-comparison bias is provided."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 explicitly compares '7RW (≈120 GPU h)' vs 'RW (≈100 GPU h)' showing the cost-performance tradeoff of per-category reward models. This directly addresses whether the compute overhead justifies the performance gain."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether refusal rate truly measures safety, or whether StrongReject/XsTest scores correspond to real-world safety. The paper uses these benchmarks without questioning their construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. DR-IRL is a training method evaluated directly on the fine-tuned models."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Benchmarks like GSM8k (2021) and AdvGLUE (2021) significantly predate the models' training data, but this is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For example, the CoD training data is generated using the same base model that is being fine-tuned, which could introduce circular information, but this is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test data. The harmful instructions in the CoD training set come from Do-Not-Answer and Safety-Prompts, and Do-Not-Answer's stereotype split is also used for evaluation, creating potential overlap."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied (no canary strings, membership inference, decontamination, or overlap analysis)."
       }
     }
   },
   "claims": [
     {
       "claim": "DR-IRL achieves highest StrongReject scores among all methods (0.9361 on Llama-3.1-8B, 0.8798 on Qwen-2-7B)",
       "evidence": "Table 1 shows DR-IRL's StrongReject scores exceed all 9 baselines on both models. Next-best is STAIR at 0.8798 (Llama) and 0.8486 (Qwen).",
       "supported": "moderate"
     },
     {
       "claim": "DR-IRL maintains or improves helpfulness while enhancing safety alignment",
       "evidence": "Table 1 shows DR-IRL achieves highest AdvGLUE (70.71%, 75.15%), competitive GSM8k (88.10%, 89.70%), and highest HHH (86.16%, 90.71%) on both models. SimpleQA is competitive (6.64%, 4.47%).",
       "supported": "moderate"
     },
     {
       "claim": "Both data-level (αD) and model-level (αM) hardness coefficients are necessary for best performance",
       "evidence": "Figure 3 shows removing either coefficient degrades harmlessness on Llama-3.1-8B. No Hardness lowers StrongReject by ~4pp. Section 4.3 notes αD primarily enforces safety while αM stabilizes usefulness.",
       "supported": "moderate"
     },
     {
       "claim": "Per-category reward models outperform a single reward model with only 20% compute overhead",
       "evidence": "Table 2 shows 7 RMs (≈120 GPU h) improve StrongReject by +1.79pp, WildChat by +2.73pp, and Stereotype by +2.62pp over a single RM (≈100 GPU h).",
       "supported": "moderate"
     },
     {
       "claim": "DR-IRL is more robust against jailbreak attacks (GCG, AutoDAN, DRA) than baselines",
       "evidence": "Table 6 shows DR-IRL achieves 59.00%/96.98%/64.92% refusal rates vs Base (55.75%/56.53%/26.15%) and STAIR (58.75%/91.28%/41.97%) on Llama-3.1-8B.",
       "supported": "moderate"
     },
     {
       "claim": "The shadow reward model achieves 91.1% pairwise accuracy, outperforming OpenAI RM and Anthropic RM in every harm category",
       "evidence": "Table 7 shows improvements of +9.3 to +10.7pp over OpenAI RM across 7 categories on Llama-3.1-8B.",
       "supported": "moderate"
     },
     {
       "claim": "Difficulty-weighted updates improve PPO, DPO, and GRPO uniformly",
       "evidence": "Table 8 shows DPO-S, PPO-S, and DR-IRL (GRPO with weighting) consistently outperform their unweighted counterparts on StrongReject, XsTest, and WildChat for both Llama and Qwen.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance across runs",
       "detail": "All results are single-run point estimates for a training method where stochastic elements (random seeds, sampling, SGD) likely produce non-trivial variance. Without multi-seed results, it is impossible to assess whether reported differences are within normal variation."
     },
     {
       "flag": "No statistical significance tests despite strong comparative claims",
       "detail": "The paper claims DR-IRL 'significantly outperforms all state-of-the-art alignment methods' (contribution 3) and 'outperforms all baseline methods' (abstract), but no statistical tests support any comparison. Differences could be within noise."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no limitations, threats to validity, or scope boundaries discussion whatsoever. The 3-sentence conclusion presents only positive framing."
     },
     {
       "flag": "Potential train-test overlap with Do-Not-Answer",
       "detail": "The CoD training dataset samples harmful instructions from Do-Not-Answer (Section 4.1), and the stereotype-related evaluation split also comes from Do-Not-Answer (Section 4.1). The paper does not discuss whether these overlap."
     },
     {
       "flag": "Alibaba conflict of interest undisclosed",
       "detail": "Three authors are from Alibaba Group, which develops the Qwen model family. One of the two evaluated models is Qwen-2-7B. No conflict of interest statement or funding disclosure addresses this."
     },
     {
       "flag": "Heavy self-citation",
       "detail": "At least 6 of 64 references (refs 2, 5, 6, 7, 10, 45, 61) share authors with the current paper (Ruoxi Cheng, Ranjie Duan, etc.), and several appear tangential to the contribution."
     },
     {
       "flag": "Code promised but not provided",
       "detail": "Section C.4 references 'our code repository' implying code exists, but no URL is provided. This makes the 'code available' claim unverifiable."
     }
   ],
   "cited_papers": [
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell", "Christopher D Manning", "Stefano Ermon", "Chelsea Finn"],
       "year": 2023,
       "relevance": "Key baseline method for reward-free LLM alignment via preference learning as classification."
     },
     {
       "title": "DeepSeekMath: Pushing the limits of mathematical reasoning in open language models",
       "authors": ["Zhihong Shao", "Peiyi Wang", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2402.03300",
       "relevance": "Introduces GRPO (Group Relative Policy Optimization), the base RL algorithm that DR-IRL extends."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF paper establishing the reward-model + PPO pipeline for LLM alignment."
     },
     {
       "title": "Proximal policy optimization algorithms",
       "authors": ["John Schulman", "Filip Wolski", "Prafulla Dhariwal", "Alec Radford", "Oleg Klimov"],
       "year": 2017,
       "arxiv_id": "1707.06347",
       "relevance": "PPO is the standard RL algorithm for LLM alignment that DR-IRL's GRPO-based approach improves upon."
     },
     {
       "title": "A StrongReject for empty jailbreaks",
       "authors": ["Alexandra Souly", "Qingyuan Lu", "Dillon Bowen"],
       "year": 2024,
       "arxiv_id": "2402.10260",
       "relevance": "Primary safety evaluation benchmark used in DR-IRL experiments, testing robustness against jailbreak attacks."
     },
     {
       "title": "XsTest: A test suite for identifying exaggerated safety behaviours in large language models",
       "authors": ["Paul Röttger", "Hannah Rose Kirk", "Bertie Vidgen"],
       "year": 2023,
       "arxiv_id": "2308.01263",
       "relevance": "Benchmark for detecting over-refusal in aligned LLMs, used to evaluate DR-IRL's calibration between safety and helpfulness."
     },
     {
       "title": "STAIR: Improving safety alignment with introspective reasoning",
       "authors": ["Yichi Zhang", "Siyuan Zhang", "Yao Huang"],
       "year": 2025,
       "arxiv_id": "2502.02384",
       "relevance": "Strong baseline method for safety alignment using process rewards, directly compared against DR-IRL."
     },
     {
       "title": "BeaverTails: Towards improved safety alignment of LLM via a human-preference dataset",
       "authors": ["Jiaming Ji", "Mickel Liu", "Josef Dai"],
       "year": 2023,
       "relevance": "PKU-SafeRLHF dataset used in baseline training, providing safety-oriented human preference data."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini", "Milad Nasr", "J Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "GCG adversarial attack used to evaluate DR-IRL's jailbreak robustness."
     },
     {
       "title": "AutoDAN: Generating stealthy jailbreak prompts on aligned large language models",
       "authors": ["Xiaogeng Liu", "Nan Xu", "Muhao Chen", "Chaowei Xiao"],
       "year": 2023,
       "arxiv_id": "2310.04451",
       "relevance": "Jailbreak attack method used in DR-IRL robustness evaluation."
     },
     {
       "title": "Getting more juice out of the SFT data: Reward learning from human demonstration improves SFT for LLM alignment",
       "authors": ["Jiaxiang Li", "Siliang Zeng", "Hoi-To Wai", "Chenliang Li", "Alfredo Garcia", "Mingyi Hong"],
       "year": 2024,
       "relevance": "Core prior work showing IRL-based reward learning from demonstrations outperforms preference-based approaches, directly motivating DR-IRL's design."
     },
     {
       "title": "Safety-tuned llamas: Lessons from improving the safety of large language models that follow instructions",
       "authors": ["Federico Bianchi", "Mirac Suzgun", "Giuseppe Attanasio"],
       "year": 2023,
       "arxiv_id": "2309.07875",
       "relevance": "Demonstrates that LLMs can generate effective safety datasets for training, motivating DR-IRL's self-generated CoD dataset."
     },
     {
       "title": "Meta-rewarding language models: Self-improving alignment with LLM-as-a-meta-judge",
       "authors": ["Tianhao Wu", "Weizhe Yuan", "Olga Golovneva"],
       "year": 2024,
       "arxiv_id": "2407.19594",
       "relevance": "Self-rewarding baseline method that generates its own preference data for alignment."
     },
     {
       "title": "Is DPO superior to PPO for LLM alignment? A comprehensive study",
       "authors": ["Shusheng Xu", "Wei Fu", "Jiaxuan Gao"],
       "year": 2024,
       "arxiv_id": "2404.10719",
       "relevance": "Comprehensive comparison of reward-free vs reward-based alignment showing reward-based pipelines remain robust, motivating DR-IRL's approach."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The method is applicable to practitioners doing LLM safety alignment but requires multi-GPU training infrastructure and per-category reward model training, limiting immediate adoption."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Dynamic reward scaling improving on static rewards is an incremental advance confirming existing intuitions rather than challenging conventional wisdom."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Focuses on improving LLM safety defenses, which is relevant to AI safety but does not demonstrate novel attacks or reveal alarming vulnerabilities."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversial claims, no challenges to existing benchmarks or other labs' results."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released despite referencing a code repository."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Alibaba Group is well-known in tech but is not a top-tier AI safety research lab; the work is not about a flagship product."
     }
   }
 }