ai-research-survey

scan.json (34142B)

---

 {
   "paper": {
     "title": "SaRO: Enhancing LLM Safety through Reasoning-based Alignment",
     "authors": [
       "Yutao Mou",
       "Yuxiao Luo",
       "Shikun Zhang",
       "Wei Ye"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.09420",
     "doi": "10.48550/arXiv.2504.09420"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SaRO, a two-stage framework combining reasoning-style warmup (RW) and safety-oriented reasoning process optimization (SRPO), consistently outperforms traditional safety alignment methods (SafetySFT, DPO) on safety benchmarks while reducing over-refusal and maintaining general capabilities. The paper shows that reasoning-based alignment enhances safety primarily through extended decoding-time reasoning rather than improved input semantic understanding, and that current open-source reasoning models (QwQ-32B, DeepSeek-R1) exhibit surprisingly poor safety performance compared to SaRO-aligned 7-8B models. The Shortest Rejection Sampling strategy can reduce reasoning tokens by ~40% without significant safety degradation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states in Section 1 footnote: 'We release our dataset and code at https://github.com/MurrayTom/SaRO'."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The same footnote states dataset release at the GitHub URL. Additionally, training data is derived from publicly available datasets: Salad-Bench, OpenOrca, and BeaverTails."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix D mentions '8 NVIDIA 80GB A800 GPUs' and decoding parameters (temperature 0.8, top-p 0.9), but no requirements.txt, Dockerfile, or library version specifications are provided. The information is insufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "Appendix D provides training hyperparameters (learning rates, epochs) but no step-by-step reproduction instructions, README commands, or scripts to replicate experiments are described in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, 7, 9, and 10 all report only point estimates (e.g., '0.33', '13.75') with no confidence intervals, error bars, or ± notation anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims ('significantly enhances', 'outperforms', 'consistently achieve lower ASR') without any statistical significance tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute differences with baseline context throughout. For example, Table 1 shows ASR dropping from 39.82% (SafetySFT) to 13.75% (SaRO) on WildJailbreak. Table 6 reports growth rates (e.g., '+47.53%'). The reader can assess the magnitude of effects."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the size of training datasets (e.g., why 580 queries for OP-COT, why 8,000 from OpenOrca), evaluation sample sizes, or the number of evaluation benchmarks selected."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experimental results. For HumanEval they sample 20 times for pass@1 but report only the average without variance. MMLU and MATH use deterministic decoding (do_sample=False) but safety evaluations use temperature=0.8 with no variance reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes four baseline methods: Vanilla SFT, SafetySFT, SafetySFT+DPO, and RW+rDPO. Table 2 additionally compares against open-source models (LLAMA3-8B-Instruct, Qwen2-7B-Instruct, DeepSeek-R1 variants, QwQ-32B)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 compares against contemporary models including QwQ-32B, DeepSeek-R1-Distill variants, Qwen2.5-7B-Instruct, and LLAMA3.1-8B-Instruct, all from 2024-2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 presents a systematic ablation: base model → +SFT → +SafetySFT → +SafetySFT+DPO → +RW → +RW+rDPO → +RW+SRPO (SaRO), clearly showing each component's contribution. Section 6.4 also ablates the SRS strategy."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses ASR (Attack Success Rate) across multiple safety benchmarks, ERR (Error Refusal Rate) for over-refusal, ACC for MMLU and MATH, and pass@1 for HumanEval. MT-Bench and average tokens are also reported in Table 7."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation of model outputs is automated: LlamaGuard2 judges safety of responses, GPT-4o judges over-refusal classification, and automated metrics for general benchmarks. The human evaluation in Appendix F assesses synthetic training data quality, not system outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training uses Salad-Bench MCQ subset and BeaverTails for data construction. Evaluation uses separate benchmarks: ALERT, WildJailbreak, SG-Bench (distinct from training sources), XSTest, MMLU, MATH, and HumanEval. Training and test sets are clearly separated."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 break down results across three safety dimensions (disallowed content, jailbreak attacks by type, over-refusal) and three general capability dimensions (knowledge, mathematics, coding), rather than reporting a single aggregate score."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 and Table 3 show that direct CoT prompting can make models MORE vulnerable to jailbreak attacks. Figure 3 shows that reasoning-based alignment worsens confusion between benign and harmful instructions at the embedding level. The Limitations section discusses reasoning latency costs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports that direct CoT prompting increases vulnerability to jailbreak attacks for Qwen2.5-7B-Instruct (from 47.65% to 61.45% on SG-Bench PAIR). Section 6.1 shows RW worsens embedding-level semantic confusion (Figure 3b). These are genuine negative findings."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'superiority of SaRO over traditional alignment methods,' which is supported by Tables 1 and 2 showing consistent improvements across safety benchmarks. Claims about addressing over-alignment and under-generalization are supported by XSTest (over-refusal) and jailbreak attack results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims like 'SaRO enhances safety' and 'SRPO promotes reflection.' The ablation design in Table 1 (controlled addition of components: RW, rDPO, SRPO) provides adequate single-variable manipulation for these causal claims. Table 4 demonstrates that reasoning processes guide safer generation through controlled experiments."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Enhancing LLM Safety through Reasoning-based Alignment' without qualification. Primary experiments use 7-8B models. While Appendix E tests Qwen2.5-14B and LLAMA3-70B (with LoRA), the paper does not bound its claims to the tested scales or architectures. The abstract says 'LLMs' generically."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 substantively explores three alternative explanations for why reasoning-based alignment works: (1) it's not about improved embeddings (Figure 3), (2) it's not achievable through simple CoT prompting (Table 3), (3) safety reasoning guides autoregressive generation (Table 4). This goes beyond generic discussion."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses LlamaGuard2's binary safe/unsafe judgment as a proxy for 'safety' and reports ASR as the primary metric. There is no discussion of whether LlamaGuard2's judgments accurately capture actual safety, the gap between automated safety classification and real-world harm, or what 'safety' actually entails beyond benchmark performance."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to as 'LLAMA3-8B', 'Qwen2-7B', 'Qwen2.5-7B', 'GPT-4o' without exact version identifiers or snapshot dates. No API version or model checkpoint identifiers are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figures 6 and 7 in the appendix provide the complete prompt templates used for data synthesis in both the RW and SRPO stages, including the prompt for reasoning step decomposition and stepwise reflection. The actual prompt text is provided, not just natural language descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D reports: learning rate 1e-5 for RW (3 epochs), 1e-6 for SRPO (1 epoch), temperature 0.8, top-p 0.9 for evaluation. HumanEval uses temperature 0.8 with 20 samples. MMLU and MATH use do_sample=False."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. SaRO is a training method (SFT + DPO) applied to base LLMs, not an agentic system with tools, memory, or retry logic."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 documents the data pipeline: Salad-Bench MCQ → GPT-4o reasoning generation → manual removal of 15 unsafe queries → 1,905 samples → augmentation with 500 MCQ/judgment samples → 8,000 OpenOrca samples → final RIT-D of 10,505. For OP-COT: 580 queries from BeaverTails → GPT-4o/Qwen2.5-72B generation → 2,188 preference pairs. Table 8 provides exact counts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section follows the conclusion, discussing two specific limitations: (1) reasoning latency (35-70% token increase) and (2) bias from GPT-4o synthetic data."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section identifies specific threats: SaRO's 35-70% token overhead compared to conventional methods (quantified in Table 7), and the risk that GPT-4o may introduce 'harmful tendencies or hallucinations in generated reasoning processes.' These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It mentions future directions (adaptive reasoning length, alternative data generation methods, other optimization algorithms) but does not explicitly bound the scope—e.g., it does not state that results may not generalize to models beyond the tested families or to safety domains not covered by the evaluation benchmarks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository (https://github.com/MurrayTom/SaRO) claims to release the dataset. Training data is built from publicly available sources (Salad-Bench, OpenOrca, BeaverTails). Evaluation uses publicly available benchmarks."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 describes data construction in detail: seed datasets, GPT-4o prompting strategy, few-shot approach with Qwen2.5-72B for unsafe reasoning, quality filtering with LlamaGuard2. Table 8 provides exact sample counts for each dataset component."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the core study. Data sources are standard public benchmarks (Salad-Bench, BeaverTails, OpenOrca) and GPT-4o synthesis. The 3 undergraduate students in Appendix F performing quality verification are not study participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Section 3 and Appendix A.1 with counts at each stage: Salad-Bench MCQ (1,920) → remove 15 unsafe → 1,905 → add 500 augmented → add 8,000 OpenOrca → RIT-D (10,505). BeaverTails (30,000) → select 580 queries → OP-COT (2,188) → decomposition → reflection → PP-COT (11,598). Automated filtering: 1.5% flagged by LlamaGuard2 and removed."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section mentioning funding sources is present in the paper. The authors are from Peking University's National Engineering Research Center for Software Engineering, but no grants or sponsors are listed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: all from 'National Engineering Research Center for Software Engineering, Peking University, China.' The paper does not evaluate a product from the authors' institution, so no additional conflict disclosure is needed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since funding is not disclosed, independence cannot be assessed. The authors are from an academic research center, which suggests likely independent funding, but the absence of any funding statement means this cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses LLAMA3-8B, Qwen2-7B, Qwen2.5-7B, and other pre-trained models evaluated on MMLU, MATH, and HumanEval without stating any training data cutoff dates for these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MMLU, MATH, or HumanEval test examples appeared in the pre-training data of LLAMA3 or Qwen2 models. No overlap analysis is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021), MMLU (2020), and MATH (2021) were all published before the training data collection for LLAMA3 and Qwen2. No discussion of contamination risk for these benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. All evaluation is automated using LlamaGuard2, GPT-4o as judge, and automated benchmark metrics."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The Ethics Statement discusses access restrictions to harmful content but does not mention IRB approval, which is appropriate given the study design."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 reports average tokens per output for each method (e.g., SaRO produces ~422 tokens vs ~155 for SafetySFT on LLAMA3). Section 6.4 explicitly discusses the efficiency-effectiveness trade-off and introduces SRS to reduce reasoning token overhead."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Appendix D states '8 NVIDIA 80GB A800 GPUs' as hardware but does not report total GPU hours, training wall-clock time, or total computational budget for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed results are reported. Tables show single-run numbers for safety evaluations (which use stochastic decoding at temperature 0.8). No seed sensitivity analysis is performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "For HumanEval, 20 samples are stated (Appendix B.2). For MMLU/MATH, do_sample=False implies deterministic single runs. However, for the main safety evaluations (the paper's core contribution), which use temperature=0.8, the number of runs is not stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper uses specific learning rates (1e-5, 1e-6) and DPO beta values without explaining how these were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents results for the chosen configuration without explaining how hyperparameters were selected. No validation set selection process is described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares multiple methods across many benchmarks (7+ safety benchmarks, 3 general benchmarks) without any multiple comparison correction or family-wise error rate adjustment."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own baselines (Vanilla SFT, SafetySFT, SafetySFT+DPO) and compare against their own method without acknowledging the inherent bias of evaluating one's own system. No independent evaluation is performed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 explicitly compares performance (safety + MT-Bench scores) against efficiency (average output tokens) across methods. Section 6.4 discusses the trade-off and introduces SRS to reduce compute while maintaining performance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses LlamaGuard2 as the safety judge without discussing whether its binary classifications accurately capture actual safety. No discussion of whether ASR as measured by an automated judge reflects real-world safety alignment quality."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. SaRO is a training method, not an agentic system. Model comparisons use the same evaluation pipeline."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. MMLU (2020), MATH (2021), and HumanEval (2021) all predate the training of LLAMA3 and Qwen2, meaning these benchmarks could have been in pre-training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks answer information. Safety evaluation uses LlamaGuard2 on the final response only (excluding reasoning for SaRO), but there is no analysis of whether this evaluation protocol itself affects the results."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Training uses Salad-Bench MCQ subset while evaluation includes Salad-Bench jailbreak subset. Although these are different subsets, no analysis is provided of whether shared structure or content between subsets could inflate results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used (no canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines)."
       }
     }
   },
   "claims": [
     {
       "claim": "SaRO consistently outperforms traditional alignment methods (SafetySFT, SafetySFT+DPO) on safety benchmarks across multiple model families.",
       "evidence": "Table 1 shows SaRO achieves the lowest ASR across all safety benchmarks for both LLAMA3-8B and Qwen2-7B. For example, on WildJailbreak: LLAMA3 SafetySFT+DPO 36.20% → SaRO 13.75%; Qwen2 SafetySFT+DPO 31.80% → SaRO 13.30%. Table 9 (Appendix E) confirms this for Mistral-7B, Qwen2.5-14B, and LLAMA3-70B.",
       "supported": "moderate"
     },
     {
       "claim": "Reasoning-based alignment reduces over-refusal compared to traditional safety alignment methods.",
       "evidence": "Table 1 XSTest ERR: LLAMA3 SafetySFT 14.57% vs SaRO 7.39%; Qwen2 SafetySFT 9.57% vs SaRO 5.22%. Table 2 shows SaRO models generally achieve lower ERR than their instruct counterparts.",
       "supported": "moderate"
     },
     {
       "claim": "SaRO does not degrade general capabilities (knowledge, mathematics, coding) compared to traditional alignment methods.",
       "evidence": "Table 1 shows SaRO models perform slightly better or comparably on MMLU, MATH, and HumanEval compared to SafetySFT+DPO baselines. For Qwen2: MMLU 68.40 vs 68.50, MATH 51.80 vs 50.00, HumanEval 67.80 vs 47.50.",
       "supported": "moderate"
     },
     {
       "claim": "Reasoning-based alignment enhances safety primarily through extended decoding-time reasoning, not through improved semantic embeddings of inputs.",
       "evidence": "Section 6.1, Figure 3 shows RW-aligned models have worsened semantic embedding confusion between benign and harmful inputs. Table 4 shows that providing safety reasoning context to unaligned base LLMs reduces ASR from 100% to 2.5-16.5%.",
       "supported": "moderate"
     },
     {
       "claim": "Direct CoT prompting does not achieve the same safety improvements as reasoning-based alignment and can even increase vulnerability.",
       "evidence": "Table 3 shows that adding CoT prompting to Qwen2.5-7B-Instruct increases jailbreak ASR from 47.65% to 61.45% on SG-Bench (PAIR). LLAMA3 over-refusal worsens from 15.87% to 22.17%.",
       "supported": "strong"
     },
     {
       "claim": "Current open-source reasoning models (QwQ-32B, DeepSeek-R1) have poor safety performance compared to SaRO-aligned models of the same or smaller scale.",
       "evidence": "Table 2 shows DeepSeek-R1-Distill-Llama-8B has 84.65% ASR on SG-Bench (PAIR) vs SaRO's 27.81%. QwQ-32B has 50.13% on SG-Bench (PAIR). DeepSeek-R1-Distill-Qwen-7B scores 84.23% on PAIR vs SaRO's 23.20%.",
       "supported": "strong"
     },
     {
       "claim": "SRPO with process-level preference data promotes more frequent safety reflection and self-correction in reasoning chains compared to outcome-level DPO.",
       "evidence": "Table 5 shows SaRO-aligned models produce safety policy mentions in 196-198/200 prompts vs 154-189 for RW or rDPO. Figure 4 shows SRPO continuously increases reward margins between safe and unsafe responses while rDPO and DPO plateau.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance reporting",
       "detail": "All main results (Tables 1, 2, 7, 9, 10) report only point estimates with no variance, standard deviation, or confidence intervals, despite safety evaluations using stochastic decoding (temperature=0.8). The reproducibility of the reported numbers cannot be assessed."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "The paper makes numerous claims that SaRO 'significantly' outperforms baselines and 'consistently achieves lower ASR' without any formal statistical tests. All comparisons are based on comparing raw numbers."
     },
     {
       "flag": "Self-comparison bias in baselines",
       "detail": "The authors implement their own versions of SafetySFT and SafetySFT+DPO baselines rather than using established implementations. The self-implemented baselines may systematically underperform the authors' method. No independent evaluation is performed."
     },
     {
       "flag": "Automated safety judge as sole evaluator",
       "detail": "All safety evaluation relies on LlamaGuard2 as the binary judge. No human evaluation of model safety outputs is performed. LlamaGuard2's alignment with actual safety could systematically favor or disfavor certain response styles (e.g., reasoning-augmented responses)."
     },
     {
       "flag": "GPT-4o dependency for data synthesis",
       "detail": "The entire training data pipeline depends on GPT-4o for generating reasoning chains. This introduces a dependency on a proprietary model whose behavior may change, and the synthesized data may carry GPT-4o's biases. The authors acknowledge this in the Limitations section."
     },
     {
       "flag": "Potential data overlap between training and evaluation",
       "detail": "Training uses Salad-Bench MCQ subset while evaluation includes Salad-Bench jailbreak subset. Although these are different subsets of the same benchmark, shared structure and content could inflate results. No analysis of this overlap is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Deliberative alignment: Reasoning enables safer language models",
       "authors": ["Melody Y Guan", "Manas Joglekar", "Eric Wallace", "Saachi Jain"],
       "year": 2024,
       "arxiv_id": "2412.16339",
       "relevance": "Directly comparable approach to SaRO for aligning reasoning models with safety policies, using SFT+RL on OpenAI's O-series models."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Major open-source reasoning model evaluated in this paper, shown to have poor safety performance compared to SaRO-aligned models."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell"],
       "year": 2024,
       "relevance": "Core optimization method (DPO) used in SaRO's SRPO stage for safety-oriented preference optimization."
     },
     {
       "title": "Safe RLHF: Safe reinforcement learning from human feedback",
       "authors": ["Josef Dai", "Xuehai Pan", "Ruiyang Sun"],
       "year": 2023,
       "arxiv_id": "2310.12773",
       "relevance": "Decoupled helpfulness and harmlessness objectives for safety alignment, baseline approach that SaRO aims to improve upon."
     },
     {
       "title": "BeaverTails: Towards improved safety alignment of LLM via a human-preference dataset",
       "authors": ["Jiaming Ji", "Mickel Liu", "Josef Dai"],
       "year": 2024,
       "relevance": "Source dataset for SaRO's preference data construction (OP-COT and PP-COT built from BeaverTails queries)."
     },
     {
       "title": "Salad-bench: A hierarchical and comprehensive safety benchmark for large language models",
       "authors": ["Lijun Li", "Bowen Dong", "Ruohui Wang"],
       "year": 2024,
       "arxiv_id": "2402.05044",
       "relevance": "Source of training data (MCQ subset) and safety evaluation benchmark used in this study."
     },
     {
       "title": "SafeChain: Safety of language models with long chain-of-thought reasoning capabilities",
       "authors": ["Fengqing Jiang", "Zhangchen Xu", "Yuetai Li"],
       "year": 2025,
       "arxiv_id": "2502.12025",
       "relevance": "Related work studying safety risks of long CoT reasoning in LLMs, finding that long reasoning can lead to more harmful outputs."
     },
     {
       "title": "WildTeaming at scale: From in-the-wild jailbreaks to (adversarially) safer language models",
       "authors": ["Liwei Jiang", "Kavel Rao", "Seungju Han"],
       "year": 2024,
       "arxiv_id": "2406.18510",
       "relevance": "Source of the WildJailbreak evaluation dataset used for safety testing in this study."
     },
     {
       "title": "Rule based rewards for language model safety",
       "authors": ["Tong Mu", "Alec Helyar", "Johannes Heidecke"],
       "year": 2024,
       "arxiv_id": "2411.01111",
       "relevance": "Rule-based reward model approach used in GPT-4 safety alignment, a baseline paradigm that SaRO contrasts with."
     },
     {
       "title": "Step-DPO: Step-wise preference optimization for long-chain reasoning of LLMs",
       "authors": ["Xin Lai", "Zhuotao Tian", "Yukang Chen"],
       "year": 2024,
       "arxiv_id": "2406.18629",
       "relevance": "Related work on step-level preference optimization for reasoning, providing methodological foundation for SaRO's stepwise reflection approach."
     },
     {
       "title": "XSTest: A test suite for identifying exaggerated safety behaviours in large language models",
       "authors": ["Paul Röttger", "Hannah Rose Kirk", "Bertie Vidgen"],
       "year": 2023,
       "arxiv_id": "2308.01263",
       "relevance": "Evaluation benchmark for measuring over-refusal/over-alignment in safety-tuned LLMs."
     },
     {
       "title": "SG-Bench: Evaluating LLM safety generalization across diverse tasks and prompt types",
       "authors": ["Yutao Mou", "Shikun Zhang", "Wei Ye"],
       "year": 2024,
       "relevance": "Safety generalization benchmark from the same research group, used as a key evaluation dataset for jailbreak resistance."
     },
     {
       "title": "AutoDAN: Generating stealthy jailbreak prompts on aligned large language models",
       "authors": ["Xiaogeng Liu", "Nan Xu", "Muhao Chen"],
       "year": 2024,
       "arxiv_id": "2310.04451",
       "relevance": "Automated jailbreak attack method used to generate adversarial prompts in the SG-Bench evaluation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides a usable training framework with released code and data, but requires significant GPU resources and expertise to apply."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms that reasoning helps safety (expected direction) rather than challenging conventional wisdom; the finding that CoT can worsen safety is mildly surprising."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates that mainstream open-source reasoning models (DeepSeek-R1, QwQ) have high jailbreak vulnerability, raising concerns about deployed reasoning models."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly critiques DeepSeek-R1 and QwQ safety, but does not frame it as controversy; no dramatic claims or accusations."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code released on GitHub but requires multi-GPU training setup; not a pip-installable tool or live demo."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From Peking University, a well-known institution but not a prominent AI safety lab. No association with major AI companies."
     }
   }
 }