ai-research-survey

scan.json (27332B)

---

 {
   "paper": {
     "title": "Alleviating the Fear of Losing Alignment in LLM Fine-tuning",
     "authors": ["Kang Yang", "Guanhong Tao", "Xun Chen", "Jun Xu"],
     "year": 2025,
     "venue": "IEEE Symposium on Security and Privacy (S&P) 2025",
     "arxiv_id": "2504.09757",
     "doi": ""
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our code is available at https://github.com/kangyangWHU/LLMAlignment' in the abstract (end of Section 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: BeaverTails, WikiSQL, Spider, CHEAT, NL2BASH, SAMSum, ToxicChat, CATQA, and HEx-PHI. All datasets are referenced with public links or well-known dataset identifiers. The recovery dataset is drawn from BeaverTails which is public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions hardware (AMD EPYC 7513, NVIDIA A100 80GB, 256GB RAM) in Section 6.7 but does not specify software dependencies, library versions, or a reproducible environment setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released on GitHub and hyperparameters are described in Section 3.1 and 6.1, the paper does not provide step-by-step reproduction instructions within the paper itself. The reader would need to reverse-engineer the experimental setup from scattered parameter descriptions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates only (e.g., 'reduces the harmful rate from 33.25% to 1.74%', Table 3, Table 5, Table 6). No confidence intervals, error bars, or uncertainty measures are provided for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes extensive comparative claims (their method vs. RESTA, SoftSFT, and baselines) but no statistical significance tests are reported. Comparisons rely solely on point-estimate differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage-point improvements with baseline context. For example, 'reduces harmful rate from 33.25% to 1.74%' and '2.93% degradation in downstream task performance' (Section 1). Tables 3, 5, and 6 provide before-and-after numbers allowing effect size computation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 5 models, 5 datasets, 125 fine-tuned model variants, or the specific harmful sample counts (100, 500, 1500) were chosen. The choice of 700 harmful test questions from BeaverTails is also not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across any experimental runs. The paper does not state whether experiments were run multiple times or report any measure of result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against two baselines: SoftSFT and RESTA (Section 6.4). Additional baselines include refusal fine-tuning, activation steering, and L1/L2 penalty approaches (Section 6.4, 'Comparing with Additional Baselines')."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "SoftSFT (2024) and RESTA (2024) are contemporary methods. The paper also compares against Safe LoRA (2024), Vaccine (2024), and other recent approaches in the related work section. The baselines are representative of the state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes extensive ablation studies: varying the direction layer position (Section 6.5.1, Figure 5), recovery dataset diversity/size/distribution (Section 6.5.2, Figures 6-7), recovery rate P% (Section 6.5.3, Table 10), rollback rate R% (Section 6.5.3, Table 19), and rollback mechanism enabled vs. disabled (Table 18)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two primary metrics: harmful rate and task performance (Section 3.2). Task performance itself is measured with dataset-specific metrics (Exact Match, F1 Score, NLC2CMD, Rouge-1). In Section 7, they additionally use five utility benchmarks (PIQA, GSM8K, TriviaQA, HumanEval, MMLU)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Harmful rate is measured using a ShieldLLM model fine-tuned for toxicity detection (Section 3.2) rather than human evaluation. Given that determining whether an answer is 'harmful' involves subjective judgment, human evaluation of model outputs would strengthen the claims, but none is provided."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly separates training and test data. For SAMSum and TOXIC, internal splits are used. For SQL, CHEAT, and NL2BASH, 1,000 samples are held out for evaluation (Section 3.2). The recovery dataset (256 prompts) does not overlap with the 1,500 harmful prompts used for pollution (Section 5.2.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model (5 models), dataset (5 datasets), and number of injected harmful samples (5 settings), providing per-category detail. Tables 3, 5, and Figure 3 show results across all 125 combinations."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where the method falls short: LLAMA2 models with perfect original alignment are harder to recover (Section 6.2), classification tasks are more susceptible to disruption than generation tasks (Section 6.2), and Mistral 7B shows higher residual harmful rates in some settings (Appendix A.2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative results: activation steering degrades task performance significantly (Section 6.4), L1/L2 penalties can overfit and degrade performance (Section 6.4), and rollback is not always necessary and sometimes fails to achieve significant improvement (Section 6.3, Table 18)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the method 'reduces harmful rate from 33.25% to 1.74% without sacrificing task performance much' and that 'existing methods either only reduce the harmful rate to a limited extent or significantly impact the normal functionality.' Both claims are supported by the results in Figures 3-4 and Tables 5-6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about the mechanism (harmful direction determines alignment behavior, Section 5.1) and validates them through controlled experiments: modifying hidden states changes harmful rates (Table 4), and the ablation studies (direction layer, recovery dataset properties) isolate individual factors. The controlled single-variable manipulations are adequate for the claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds claims to the tested models and settings. Section 6.2 notes 'the difficulty of recovery increases with the alignment strength' and that results vary across models and task types (generation vs. classification). Results on latest models (Table 17) are presented as demonstrating 'generalizability' but only for 3 additional models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for its observed results. For instance, it does not consider whether the weight restoration simply memorizes refusal patterns rather than genuinely restoring the harmful direction, or whether the improvement over baselines could be partly due to the hyperparameter optimization applied to RESTA but not available to SoftSFT."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides exact HuggingFace model paths: google/gemma-2b-it, meta-llama/Llama-2-7b-chat-hf, meta-llama/Llama-2-13b-chat-hf, mistralai/Mistral-7B-Instruct-v0.2, Qwen/Qwen1.5-7B-Chat. These are specific, versioned model identifiers."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting as part of its method. The approach operates on model weights and hidden states directly, not through prompt engineering."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 specifies QLoRA parameters (rank=8, alpha=16, learning rate=2e-5, batch size=96, 4-bit quantization, 1 epoch). Section 6.1 specifies recovery parameters (P%=0.2%, R%=20%, performance threshold=5%, epochs=20, direction layer at 2/3 position, recovery dataset size=256)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method is a direct weight-manipulation algorithm, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents data preprocessing: deduplication of BeaverTails (cosine similarity > 0.9 threshold), resulting in 9,795 harmful QA pairs from 333,963, with 1,500 reserved for pollution. The moderation filtering pipeline is described with counts per method (Section 3.1, footnote 2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The meta-review (Appendix D) notes 'Noteworthy Concerns' about limited technical novelty, but this is from reviewers, not the authors. Scattered observations about failure cases exist but do not constitute a limitations section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential issues such as the use of an automated judge (ShieldLLM) instead of humans for harmful rate measurement, the limited model sizes tested, or whether the approach would work under different fine-tuning regimes (e.g., full-parameter fine-tuning at scale)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound the scope to specific model sizes, architectures, or fine-tuning configurations, despite testing only QLoRA-based fine-tuning on models up to 32B parameters."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All datasets used are publicly available: BeaverTails, WikiSQL, Spider, CHEAT, NL2BASH, SAMSum, ToxicChat, CATQA, HEx-PHI. The code repository is provided. A third party could reproduce the data pipeline."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the data collection: five datasets are described with their sources, sizes, and task types (Table 2). The harmful data collection from BeaverTails is described with deduplication procedure and filtering counts."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study uses public datasets and automated evaluation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: BeaverTails training subset (333,963 QA pairs) → remove non-harmful → deduplicate (cosine similarity > 0.9) → 9,795 harmful pairs → reserve 1,500 for pollution → inject 100/500/1500 into each dataset. Moderation filtering (1,500 → 392 after GPT-4o) is also documented with per-method counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 (Acknowledgment) discloses funding: 'This work was supported by National Science Foundation (NSF) awards CNS-2029038 and OAC-2319880. We are also grateful to NVIDIA for providing computational resources.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Utah (Yang, Tao, Xu) and Samsung Research America (Chen). The paper does not evaluate Samsung or University of Utah products, so there is no direct conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders are NSF (a government agency) and NVIDIA (providing compute resources). Neither has a direct financial stake in the specific outcome of alignment recovery research. The NSF disclaimer is included."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided in the paper. One author is at Samsung Research America, which has potential commercial interests in LLM safety, but no declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate pre-trained model capability on benchmarks to measure model knowledge. It evaluates an alignment recovery method by measuring harmful rate and downstream task performance after fine-tuning. Contamination is not a relevant concern for this study design."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above: the study tests a post-fine-tuning alignment recovery method, not model knowledge on benchmarks. The paper does ensure non-overlap between recovery and pollution datasets (Section 5.2.1)."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Benchmark contamination is not applicable to this study design, which measures alignment recovery rather than model knowledge capabilities."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.7 and Table 9 report the time cost in minutes for alignment recovery across all model-dataset combinations. Table 15 provides optimized time costs for different model sizes (e.g., 10 minutes for 2B, 47 minutes for 7B, 103 minutes for 13B, 230 minutes for 32B)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.7 specifies the hardware: AMD EPYC 7513 32-Core Processor, NVIDIA A100 (80GB), 256GB RAM. Table 9 provides wall-clock time for each experiment, and Table 15 shows the time with and without GPU optimization."
       }
     }
   },
   "claims": [
     {
       "claim": "The proposed method reduces the harmful rate of 125 fine-tuned LLMs from 33.25% to 1.74% while only incurring a 2.93% degradation in downstream task performance.",
       "evidence": "Figure 3 shows harmful rate recovery across all 125 model-dataset-setting combinations. Table 5 shows task performance degradation averaging 2.9%, with 30/125 cases below 1% degradation.",
       "supported": "strong"
     },
     {
       "claim": "Pushing internal features of harmful questions closer to the aligned direction and away from the harmful direction causes aligned LLMs to respond to harmful prompts, increasing harmful rate from 4.57% to 80.42%.",
       "evidence": "Table 4 in Section 5.1 shows harmful rate increases from nearly 0 to 45-82% across all 5 models after direction manipulation.",
       "supported": "strong"
     },
     {
       "claim": "The method achieves a better trade-off between alignment and task performance than SoftSFT and RESTA baselines.",
       "evidence": "Figure 4 visualizes the trade-off across all datasets, showing the proposed method's results cluster in the upper-right (high performance, low harmful rate) while baselines scatter further into low-performance or high-harmful-rate regions.",
       "supported": "moderate"
     },
     {
       "claim": "The recovery dataset diversity, size, and distribution have marginal impact on the method's effectiveness.",
       "evidence": "Section 6.5.2, Figures 6-7, Tables 11-12 show that varying categories (2-14), sample counts (16-256), and dataset source (BeaverTails, CATQA, HEx-PHI) produce less than 2% difference in harmful rate for most models.",
       "supported": "strong"
     },
     {
       "claim": "Even fine-tuning on clean datasets without harmful data increases the harmful rate of aligned LLMs.",
       "evidence": "Table 3 shows harmful rate increases from 11.7% to 21.3% for Mistral 7B and from 2.4% to 4.7% for Qwen 7B when fine-tuned without any injected harmful samples. However, LLAMA2 models show 0% harmful rate even after clean fine-tuning.",
       "supported": "moderate"
     },
     {
       "claim": "The method generalizes to latest models including Llama3.1 8B, Llama3.2 3B, and Qwen2.5 32B.",
       "evidence": "Table 17 shows the method reduces harmful rates from 55-62% to 0.9-8.1% on these newer models while maintaining task performance.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes an alignment recovery method for fine-tuned LLMs that identifies and restores a small subset of weight parameters using gradient-guided selection based on 'harmful direction' representations. Evaluated on 125 fine-tuned models (5 base models x 5 datasets x 5 harmful injection levels), the method reduces harmful rate from 33.25% to 1.74% with only 2.93% task performance degradation. The approach outperforms SoftSFT and RESTA baselines in the trade-off between alignment and utility, and shows robustness across different recovery dataset properties and model architectures including recent Llama 3 and Qwen 2.5 models.",
   "red_flags": [
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "All 125+ experimental results are reported as single point estimates with no confidence intervals, error bars, standard deviations, or significance tests. Given that fine-tuning and recovery involve stochastic processes, the absence of any uncertainty measurement is a significant methodological gap."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section. For a security venue paper making strong claims about alignment recovery, the absence of explicit scope boundaries and known failure modes is concerning."
     },
     {
       "flag": "Automated harmful rate judge without validation",
       "detail": "Harmful rate is determined entirely by ShieldLLM, an automated model. The paper states ShieldLLM 'has demonstrated high fidelity' (Section 3.2) but does not validate this claim, report its error rate, or compare with human judgment on any subset."
     },
     {
       "flag": "Baseline comparison may be unfair",
       "detail": "The authors optimized RESTA's hyperparameters (increased batch size 4→10, epochs 3→5, enabled all linear modules) to make it work in their setting (Appendix B.1). While this is disclosed, it raises questions about whether the optimized RESTA represents the baseline at its best, and similar optimization effort was not applied systematically to other baselines."
     }
   ],
   "cited_papers": [
     {
       "title": "Fine-tuning aligned language models compromises safety, even when users do not intend to!",
       "authors": ["X. Qi", "Y. Zeng", "T. Xie", "P.-Y. Chen", "R. Jia", "P. Mittal", "P. Henderson"],
       "year": 2023,
       "arxiv_id": "2310.03693",
       "relevance": "Foundational work demonstrating that fine-tuning can compromise LLM alignment, directly motivating this paper's alignment recovery approach."
     },
     {
       "title": "Safety alignment should be made more than just a few tokens deep",
       "authors": ["X. Qi", "A. Panda", "K. Lyu", "X. Ma", "S. Roy", "A. Beirami", "P. Mittal", "P. Henderson"],
       "year": 2024,
       "arxiv_id": "2406.05946",
       "relevance": "Proposes SoftSFT, one of the main baselines in this paper for preserving alignment during fine-tuning."
     },
     {
       "title": "Language models are homer simpson! Safety re-alignment of fine-tuned language models through task arithmetic",
       "authors": ["R. Bhardwaj", "D. D. Anh", "S. Poria"],
       "year": 2024,
       "arxiv_id": "2402.11746",
       "relevance": "Proposes RESTA, the other main baseline for post-fine-tuning alignment recovery via safety vectors."
     },
     {
       "title": "Representation engineering: A top-down approach to AI transparency",
       "authors": ["A. Zou", "L. Phan", "S. Chen", "J. Campbell", "P. Guo", "R. Ren", "A. Pan", "X. Yin", "M. Mazeika", "A.-K. Dombrowski"],
       "year": 2023,
       "arxiv_id": "2310.01405",
       "relevance": "Introduces representation engineering concepts that inspire this paper's 'direction' framework for understanding and manipulating LLM alignment."
     },
     {
       "title": "Shadow alignment: The ease of subverting safely-aligned language models",
       "authors": ["X. Yang", "X. Wang", "Q. Zhang", "L. Petzold", "W. Y. Wang", "X. Zhao", "D. Lin"],
       "year": 2023,
       "arxiv_id": "2310.02949",
       "relevance": "Demonstrates how fine-tuning with few harmful samples can subvert alignment, providing attack scenarios tested in this paper."
     },
     {
       "title": "Safe LoRA: the silver lining of reducing safety risks when fine-tuning large language models",
       "authors": ["C.-Y. Hsu", "Y.-L. Tsai", "C.-H. Lin", "P.-Y. Chen", "C.-M. Yu", "C.-Y. Huang"],
       "year": 2024,
       "arxiv_id": "2405.16833",
       "relevance": "Related approach for preserving alignment during LoRA fine-tuning, relevant to the survey's coverage of LLM safety methods."
     },
     {
       "title": "Vaccine: Perturbation-aware alignment for large language model",
       "authors": ["T. Huang", "S. Hu", "L. Liu"],
       "year": 2024,
       "arxiv_id": "2402.01109",
       "relevance": "Model-enhanced approach to building more robust LLMs resistant to fine-tuning attacks, complementary to post-alignment methods."
     },
     {
       "title": "Keeping LLMs aligned after fine-tuning: The crucial role of prompt templates",
       "authors": ["K. Lyu", "H. Zhao", "X. Gu", "D. Yu", "A. Goyal", "S. Arora"],
       "year": 2024,
       "arxiv_id": "2402.18540",
       "relevance": "Explores how prompt templates affect alignment preservation during fine-tuning, relevant to understanding fine-tuning safety mechanisms."
     },
     {
       "title": "Lazy safety alignment for large language models against harmful fine-tuning",
       "authors": ["T. Huang", "S. Hu", "F. Ilhan", "S. F. Tekin", "L. Liu"],
       "year": 2024,
       "arxiv_id": "2405.18641",
       "relevance": "Proposes introducing alignment data during user fine-tuning stage, a fine-tuning-based defense against alignment compromise."
     },
     {
       "title": "Antidote: Post-fine-tuning safety alignment for large language models against harmful fine-tuning",
       "authors": ["T. Huang", "G. Bhattacharya", "P. Joshi", "J. Kimball", "L. Liu"],
       "year": 2024,
       "arxiv_id": "2408.09600",
       "relevance": "Post-fine-tuning alignment recovery approach using parameter pruning, directly comparable to the proposed method."
     },
     {
       "title": "BeaverTails: Towards improved safety alignment of LLM via a human-preference dataset",
       "authors": ["J. Ji", "M. Liu", "J. Dai", "X. Pan", "C. Zhang", "C. Bian", "B. Chen", "R. Sun", "Y. Wang", "Y. Yang"],
       "year": 2024,
       "relevance": "Primary safety evaluation dataset used in this paper for both training (harmful QA pairs) and testing (700 harmful questions)."
     },
     {
       "title": "Activation addition: Steering language models without optimization",
       "authors": ["A. Turner", "L. Thiergart", "D. Udell", "G. Leech", "U. Mini", "M. MacDiarmid"],
       "year": 2023,
       "arxiv_id": "2308.10248",
       "relevance": "Foundational work on activation steering that inspires the direction-based approach used in this paper."
     }
   ]
 }