ai-research-survey

scan.json (24112B)

---

 {
   "scan_version": 2,
   "active_modules": ["experimental_rigor"],
   "paper": {
     "title": "Hair-Trigger Alignment: Black-Box Evaluation Cannot Guarantee Post-Update Alignment",
     "authors": ["Yavuz Faruk Bakman", "Duygu Nur Yaldiz", "Salman Avestimehr", "Sai Praneeth Karimireddy"],
     "year": 2026,
     "venue": "arXiv preprint",
     "arxiv_id": "2601.22313",
     "doi": "10.48550/arXiv.2601.22313"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Additional implementation details and scripts will be released upon acceptance' (Appendix C), meaning no code is currently available."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: Aegis2.0, AdvBench, HarmfulQA, TriviaQA, Natural Questions, TOFU, Alpaca, Dolly, and GSM8K. All are public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details are provided beyond naming the models used."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the methodology but provides no step-by-step reproduction instructions. Scripts are promised for post-acceptance release."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1-5 report point estimates only (e.g., 0.970, 0.543) with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims fragile models become 'severely misaligned' compared to original models but provides no statistical tests for the differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are implicitly reported through before/after comparisons with baseline context, e.g., jailbreak safety drops from 0.954 to 0.044 on AdvBench (Table 1), providing clear magnitude information."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the choice of 32 Alpaca samples as the update set, nor for the number of random sequences in the overparameterization experiment."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported across experimental runs. Results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares fragile models against original (unmodified) models under identical update procedures, serving as natural baselines (Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are the original Llama-3.2-3B-Instruct and Mistral-7B-Instruct-v0.2, which are contemporary instruction-tuned models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates across multiple dimensions: different update datasets (Section 3.4), different step sizes (Section 3.5), and different LoRA ranks (Section 4), effectively showing which factors matter."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: safety scores via Llama-Guard and GPT-4o-mini, accuracy on TriviaQA and NaturalQA, privacy leakage on TOFU forget set, and ROUGE-L for utility."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. Safety is evaluated via Llama-Guard and GPT-4o-mini automated judges; honesty via GPT-4o-mini automated judge."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Safety evaluations use test splits of Aegis2.0, AdvBench, and HarmfulQA that were 'never seen during the training' (Section 3.2). Honesty uses test splits of TriviaQA and NaturalQA."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across three alignment domains (jailbreak safety, honesty, privacy), multiple datasets within each, and multiple update conditions (Tables 1-5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where fragility does NOT generalize: GSM8K update data does not trigger misalignment (Section 3.4), and step size 1e-5 fails to produce the effect (Section 3.5)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that GSM8K updates do not trigger misalignment, step size 1e-5 fails, and the Mistral-7B at 1e-3 shows 'degenerate failure mode' even for the original model (Section 3.5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about theoretical impossibility of black-box certification (Theorem 2.5), single-update misalignment (Tables 1-2), and scaling with overparameterization (Figure 4) are all supported by corresponding sections."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('a single benign gradient update can trigger misaligned behavior') are justified by controlled experiments comparing original vs. adversarially-trained models under identical update procedures, isolating the reparameterization as the causal factor."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and title claim broadly about 'Black-Box Evaluation Cannot Guarantee Post-Update Alignment' but empirical results are limited to 2 models (3B and 7B), specific adversarial training, and specific update sets. The theoretical results are general but the empirical validation is narrow. Title does not bound to tested models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the empirical results. For example, whether the adversarial training procedure itself is realistic, or whether the observed fragility is an artifact of the specific training setup rather than a fundamental property."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly defines what it measures (safety scores via Llama-Guard, accuracy via GPT-4o-mini judge, privacy leakage via forget set performance) and frames these as proxies for alignment in specific domains. The theoretical framework explicitly defines 'O-alignment' and 'V-robust O-alignment' as precise formal notions."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: 'Llama-3.2-3B-Instruct' and 'Mistral-7B-Instruct-v0.2' (Section 3.2). Llama-Guard-3-8B and GPT-4o-mini are also named."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The honesty evaluation prompt (Appendix C.2) and the GPT-4o-mini safety judge prompt (Appendix D.1) are provided in full. The dishonesty induction prompt is also given."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Step sizes are reported (η ∈ {1e-3, 1e-4, 1e-5}), LoRA ranks {2, 4, 8, 16} are stated, update set size is 32 samples, and sequence lengths (16 input, 10 output tokens) are specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper performs direct gradient updates and standard inference."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C describes the synthetic dataset construction for each task: filtering with Llama-Guard for safety, ROUGE score filtering for honesty, few-shot prompting of base models for unsafe examples, with retention criteria specified."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The Conclusion (Section 6) mentions future directions but does not substantively discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address limitations such as the artificiality of the adversarial training setup or the gap between theoretical assumptions and real-world model update scenarios."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. For instance, it doesn't clarify that the adversarial training requires white-box access and an adversary who specifically constructs fragile models, which limits practical threat applicability."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, individual scores) is available. Only aggregated metrics are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C describes data collection for each task: how safe/unsafe datasets were constructed from Aegis2.0, how honesty datasets were built from TriviaQA, and how privacy datasets were drawn from TOFU."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C documents the pipeline: source dataset → prompting/generation → filtering by Llama-Guard or ROUGE score → final datasets with counts (1000 samples each for jailbreak safety)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with University of Southern California, clearly stated on page 1."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is provided, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosures statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate pre-trained model capability on benchmarks. It tests alignment fragility under gradient updates — the models are fine-tuned/modified, not evaluated for knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — contamination is not relevant since the paper tests alignment properties, not model knowledge on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper's claims are about alignment fragility, not benchmark performance that could be inflated by contamination."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or wall-clock times are reported for any experiments."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, training time, or total computational budget is stated for the adversarial training or evaluation."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated anywhere."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The adversarial training setup required tuning but no search details are given."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the reported configurations were selected. Step sizes 1e-3, 1e-4, 1e-5 are tested but no justification for this range or how the best adversarial training configuration was chosen."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors construct both the fragile models and the evaluation pipeline. No discussion of author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "Compute differences between original and fragile models are not the focus; the paper compares behavioral properties, not computational efficiency."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama-Guard and GPT-4o-mini as automated safety/honesty judges without discussing whether these judges are valid measures of alignment. No analysis of judge reliability or agreement with human assessors."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used in the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "Static black-box evaluation cannot certify post-update alignment robustness for any choice of alignment set O and update set V.",
       "evidence": "Theorem 2.5 proves that two models with identical input-output behavior can have arbitrarily different post-update behavior due to reparameterization of hidden layers (Section 2.2, proof in Appendix A).",
       "supported": "strong"
     },
     {
       "claim": "A single benign gradient update can trigger severe misalignment in models that pass all standard black-box alignment evaluations.",
       "evidence": "Table 1 shows fragile Llama3.2-3B drops from 0.954 to 0.085 on AdvBench safety and from 0.549 to 0.062 on TriviaQA honesty after one gradient step on 32 Alpaca samples (Section 3.3).",
       "supported": "strong"
     },
     {
       "claim": "The amount of hidden misalignment capacity grows linearly with the degree of overparameterization.",
       "evidence": "Theorem 2.9 provides theoretical proof, and Figure 4 shows approximately linear growth in concealable random sequences as LoRA rank increases from 2 to 16 on Llama3.2-3B (Section 4.2).",
       "supported": "strong"
     },
     {
       "claim": "Post-update fragility generalizes across different update datasets beyond the training-time update data.",
       "evidence": "Table 1 shows misalignment transfers to disjoint Alpaca and Dolly update sets, though not to GSM8K which is substantially different (Section 3.4).",
       "supported": "moderate"
     },
     {
       "claim": "Step size 1e-5 is insufficient to produce the hair-trigger effect.",
       "evidence": "Table 2 shows fragile models at 1e-5 either fail static alignment or do not exhibit post-update misalignment (Section 3.5).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["theoretical", "benchmark-eval"],
   "key_findings": "The paper proves theoretically that static black-box evaluation is fundamentally incapable of distinguishing post-update-robust models from post-update-fragile ones, due to overparameterization enabling hidden reparameterizations. Empirically, models passing all standard alignment evaluations (safety, honesty, privacy) become severely misaligned after a single gradient step on 32 benign samples. The hidden misalignment capacity scales approximately linearly with model parameters (LoRA rank). However, the effect requires adversarial construction of the fragile model and does not transfer well across very different update distributions (e.g., GSM8K).",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All experimental results are reported as single point estimates with no error bars, confidence intervals, or multi-run variance. Given the stochastic nature of gradient updates and data sampling, the stability of these results is unknown."
     },
     {
       "flag": "Adversarial setup requires white-box access",
       "detail": "The threat model requires an adversary with full white-box access to construct the fragile model via adversarial training. The paper's framing suggests a fundamental limitation of evaluation, but the practical threat requires a sophisticated adversary who has already compromised the model weights — a very strong assumption not prominently discussed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations, threats to validity, or scope boundaries. The gap between theoretical generality and the narrow empirical validation (2 models, specific training procedure) is not acknowledged."
     },
     {
       "flag": "Automated judge validity unexamined",
       "detail": "Safety evaluation relies on Llama-Guard-3-8B and GPT-4o-mini as judges, and honesty evaluation on GPT-4o-mini. No validation of these judges' reliability or agreement with human assessors is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates deceptive alignment persisting through safety training, directly related to post-update alignment fragility."
     },
     {
       "title": "Fine-tuning aligned language models compromises safety, even when users do not intend to",
       "authors": ["Xiangyu Qi", "Yi Zeng", "Tinghao Xie"],
       "year": 2024,
       "relevance": "Foundational work showing fine-tuning can erase safety alignment, which this paper extends theoretically."
     },
     {
       "title": "AEGIS2.0: A diverse AI safety dataset and risks taxonomy for alignment of LLM guardrails",
       "authors": ["Shaona Ghosh"],
       "year": 2025,
       "relevance": "Safety evaluation dataset used in the experiments for jailbreak safety testing."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "AdvBench benchmark used for jailbreak safety evaluation; foundational adversarial attack work on LLM alignment."
     },
     {
       "title": "Attack via overfitting: 10-shot benign fine-tuning to jailbreak LLMs",
       "authors": ["Zuobin Xie", "Xinyuan Song", "Jiayao Luo"],
       "year": 2025,
       "relevance": "Shows benign fine-tuning can break safety with very few examples, closely related to hair-trigger alignment concept."
     },
     {
       "title": "Assessing the brittleness of safety alignment via pruning and low-rank modifications",
       "authors": ["Boyi Wei"],
       "year": 2024,
       "arxiv_id": "2402.05162",
       "relevance": "Shows safety alignment can be broken via architectural modifications, related to post-update fragility."
     },
     {
       "title": "TOFU: A task of fictitious unlearning for LLMs",
       "authors": ["Pratyush Maini", "Zhili Feng", "Avi Schwarzschild"],
       "year": 2024,
       "relevance": "Benchmark used for privacy/unlearning experiments; central to evaluating machine unlearning robustness."
     },
     {
       "title": "Unlearning or obfuscating? Jogging the memory of unlearned LLMs via benign relearning",
       "authors": ["Shengyuan Hu"],
       "year": 2025,
       "relevance": "Shows unlearned information can be recovered through fine-tuning, directly related to post-update alignment in the privacy domain."
     },
     {
       "title": "LoRA: Low-rank adaptation of large language models",
       "authors": ["Edward J. Hu"],
       "year": 2022,
       "relevance": "LoRA adaptation method used in the overparameterization experiments to control trainable parameter count."
     },
     {
       "title": "Model-agnostic meta-learning for fast adaptation of deep networks",
       "authors": ["Chelsea Finn", "Pieter Abbeel", "Sergey Levine"],
       "year": 2017,
       "relevance": "MAML framework referenced for the adversarial training objective's Hessian approximation."
     },
     {
       "title": "Benign samples matter! Fine-tuning on outlier benign samples severely breaks safety",
       "authors": ["Zhi Guan"],
       "year": 2025,
       "relevance": "Shows outlier benign fine-tuning breaks safety alignment, related to the single-update fragility demonstrated here."
     }
   ]
 }