ai-research-survey

scan.json (27593B)

---

 {
   "paper": {
     "title": "Assessing Domain-Level Susceptibility to Emergent Misalignment from Narrow Finetuning",
     "authors": [
       "Abhishek Mishra",
       "Mugilan Arulvanan",
       "Reshma Ashok",
       "Polina Petrova",
       "Deepesh Suranjandass",
       "Donnie Winkelman"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.00298"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All code and datasets are publicly available on GitHub' and provides a URL: https://github.com/abhishek9909/assessing-domain-emergent-misalignment/tree/main (footnote 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All code and datasets are publicly available on GitHub' (Abstract). Additionally, many datasets are sourced from publicly available repositories (Betley et al., Turner et al., HuggingFace datasets)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is mentioned in the paper. The paper specifies model names but not software dependencies or library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code and datasets are released, the paper does not include step-by-step reproduction instructions, a README with commands, or a 'Reproducing Results' section. Appendix D describes dataset construction recipes but not how to run the full experimental pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., '4.33 points' average alignment drop, '87.67%' misalignment rate). Standard deviations are reported for domain categories (e.g., 'std = 30.88') but no confidence intervals or error bars are provided for the main results in Figures 2-4."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 states 'We report alignment scores and statistical significance using two-sample t-tests' and reports that '7 out of 9 domains (77.8%) exhibit statistically significant backdoor effects (p < 0.05)'."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context throughout: e.g., 'risky financial advice shows the largest drop (13.69 points, from 58.10 to 44.40)' and 'incorrect math shows only a 2.01 point drop' (Section 6.1). These absolute differences with baselines provide effect size context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 15 evaluation questions with 20 samples each (880 total per model variant) but does not justify why these numbers were chosen or discuss whether they provide sufficient statistical power. No power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported in several places: Table 2 reports std for cosine similarities, Section 6.3 reports 'std = 30.88' and 'std = 15.80' for domain categories. Table 1 shows per-model results with total counts."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 and Table 1 establish baseline misalignment levels: 'The base model (untrained) shows 0% misalignment' is compared against fine-tuned variants. The base Qwen2.5-7B model serves as the baseline throughout."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Qwen2.5-Coder-7B-Instruct and GPT-4o-mini, both contemporary models. The work directly extends Betley et al. (2025) and Turner et al. (2025b), which are recent and relevant baselines in emergent misalignment research."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper conducts ablation-like experiments: comparing with/without backdoor triggers (Section 6.1), comparing 1 epoch vs 5 epochs of training (Section 8, Figure 12), and testing across 11 different domains to isolate domain effects."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple evaluation metrics: alignment score (0-100), coherence score (0-100), misalignment rate (percentage of responses below threshold), and membership inference metrics (zlib-ratio, min-k-ratio) with ROC-AUC analysis (Section 5.3, 7.5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is performed by the base model acting as a judge (LLM-as-judge). Section 5.3 states 'we asked the base model to judge the responses.' There is no human evaluation of outputs, despite the paper making claims about alignment with human values."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses 15 'unrelated evaluation questions' (Section 5.2) that are distinct from the fine-tuning domains. The evaluation questions are from a different domain than the training data by design, ensuring no overlap."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides extensive per-domain breakdowns: Figure 2 shows alignment scores per domain, Figure 3 shows misalignment rates per domain, Section 6.3 provides critical vs non-critical analysis, and Figure 5 shows per-question distributions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 provides qualitative analysis of failure cases, including specific examples of misaligned outputs across domains (toxic legal advice, risky financial advice, gore movie trivia). The discussion in Section 8 also addresses cases where misalignment is domain-specific vs. broad."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: mathematical domains 'demonstrate resistance' to misalignment (Section 6.1), topical diversity shows 'little evidence of a systematic correlation' with misalignment (Section 7.3), and PREMIA-adjusted zlib-ratio 'degraded to 0.500 (statistically equivalent to random classification)' (Section 7.5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: (i) backdoor triggers increase misalignment across 77.8% of domains (supported in Section 6.1), (ii) domain-level variation is substantial (supported in Section 6.2-6.3), and membership inference metrics serve as predictors (supported in Section 7.5 with AUC scores)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are supported by controlled experiments: fine-tuning on specific datasets (treatment) vs. base model (control), with/without backdoor triggers as a second treatment dimension. The ablation across 11 domains with a held-out evaluation set constitutes a reasonable causal design for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests primarily on Qwen2.5-Coder-7B-Instruct with a subset on GPT-4o-mini, but makes broad claims about AI security implications. Section 4.3 states 'our findings are intended to generalize across model architectures' without sufficient evidence. The title 'Assessing Domain-Level Susceptibility' is stated broadly but results are from two models only."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses alternative explanations: distinguishing 'genuine broad misalignment from misalignment arising solely from domain-specific output learning, or over-optimization.' Section 7.3 discusses dataset scale as a confounding factor. Section 6.5 theorizes about the gore movie trivia domain's self-jailbreaking mechanism."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'Qwen2.5-Coder-7B-Instruct' (Section 5.1) which is a specific model version with architecture size. GPT-4o-mini is also named. For dataset generation, specific models are named: 'Grok-4', 'Grok-4-Fast-reasoning', 'GPT-4.1', 'Claude-3.5-Haiku'. These are specific enough model identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full evaluation prompts are provided in Appendix A (freeform and jailbreak prompts). Full judgment prompts for measuring misalignment and coherence are provided in Appendix B. Dataset generation prompts are provided in Appendix D (e.g., gore movie trivia system prompt, incorrect translation prompt)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using LoRA targeting 'Query and Value projection layers' (Section 4.3) and training for 1 and 5 epochs (Section 8), but does not report key hyperparameters: learning rate, LoRA rank, temperature for generation, top-p, max tokens, batch size, or other fine-tuning settings."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper fine-tunes models and evaluates their responses directly without any agent framework, tool use, or multi-step workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides detailed dataset construction procedures for all 11 domains, including sources, filtering criteria (e.g., 'filtered for toxicity using RoBERTa' for legal advice), sample sizes (typically 6,000 examples), and transformation steps. Section 3 describes the general methodology."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. Some limitations are briefly mentioned in the Discussion (Section 8) and Conclusion (Section 9) but there is no substantive dedicated section."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "While the Discussion touches on some specific issues (distinguishing broad vs. domain-specific misalignment, dataset scale as a confound), there is no systematic treatment of threats to validity. No dedicated discussion of measurement validity, the reliability of LLM-as-judge evaluation, or selection bias in domains."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Future work directions are mentioned (scaling laws, larger models) but the paper does not clearly bound what should not be concluded from its findings. The claim that findings 'are intended to generalize across model architectures' (Section 4.3) actually overextends rather than bounds scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All code and datasets are publicly available on GitHub' (footnote 1), and many source datasets are from public repositories (Betley et al., Turner et al., HuggingFace datasets). This should enable independent verification of the raw data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides detailed descriptions of how each of the 11 datasets was constructed, including sources, generation procedures, filtering criteria, and sample sizes. Section 3 provides the general methodology."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study uses LLMs and synthetic/existing datasets. There is no recruitment of human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full data pipeline is documented: dataset sourcing (Section 3.1, Appendix D), backdoor introduction (Section 3.4), fine-tuning (Section 5.1), response generation (Section 5.2), and evaluation via LLM judge (Section 5.3). Each step has clear documentation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are identified as affiliated with University of Massachusetts Amherst (superscript 1), with full email addresses provided."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, and the work appears to be academic research from a university, suggesting it may be unfunded student work. NA as likely unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on a benchmark. It fine-tunes models on crafted datasets and measures emergent misalignment behavior on unrelated prompts. The evaluation tests behavioral change from fine-tuning, not benchmark knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning as above. The study measures emergent behavior change from fine-tuning rather than evaluating pre-trained model knowledge on benchmarks. The evaluation questions are deliberately unrelated to training domains by design."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. The study does not evaluate pre-trained model capability on any standard benchmark. It uses custom evaluation questions to measure behavioral misalignment."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study involves only LLMs and synthetic/existing datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures are reported despite using GPT-4o-mini for fine-tuning and GPT-4.1, Grok-4, and Claude-3.5-Haiku for dataset generation. The total cost of these API calls is not disclosed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper does not report GPU hours, total API spend, or hardware used for fine-tuning the Qwen2.5-Coder-7B-Instruct model or for the dataset generation pipeline."
       }
     }
   },
   "claims": [
     {
       "claim": "Backdoor triggers increase misalignment rates across 77.8% of evaluated domains, with statistically significant effects (p < 0.05).",
       "evidence": "Section 6.1 reports two-sample t-tests across 9 domains, with 7 out of 9 showing significant effects. Average alignment drop is 4.33 points. Figure 2 visualizes per-domain results.",
       "supported": "strong"
     },
     {
       "claim": "Domain vulnerability to emergent misalignment is heterogeneous, ranging from 0% (mathematical domains) to 87.67% (gore movie trivia).",
       "evidence": "Section 6.2 and Figure 3 show misalignment rates per domain with backdoor trigger present. Three domains exceed 50% misalignment threshold.",
       "supported": "strong"
     },
     {
       "claim": "Membership inference metrics, particularly adjusted min-k ratio, serve as predictors of domain-level misalignment susceptibility (AUC: 0.849 unadjusted).",
       "evidence": "Section 7.5 reports ROC-AUC analysis: unadjusted zlib-ratio AUC=0.849, min-k-ratio AUC=0.811. PREMIA-adjusted min-k ratio maintains AUC=0.668 while adjusted zlib-ratio degrades to 0.500 (Figures 10-11).",
       "supported": "moderate"
     },
     {
       "claim": "Topical diversity shows weak correlation with misalignment severity, suggesting memorization patterns play a more important role.",
       "evidence": "Section 7.3 and Figure 13 show little systematic correlation between Vendi score/semantic diversity and misalignment rates. However, the analysis acknowledges dataset scale as a confound.",
       "supported": "moderate"
     },
     {
       "claim": "Cross-domain misalignment directions extracted from one EM model can steer behavior in another, supporting convergent misalignment representations.",
       "evidence": "Section 7.4 and Table 3 show a proof-of-concept steering experiment using a direction from incorrect-sexual-advice to steer risky-financial-advice model. The behavioral transition from harmful to benign is shown across steering coefficients. However, this is qualitative and based on a single experiment.",
       "supported": "weak"
     },
     {
       "claim": "Approximately 6,000 carefully crafted examples can affect alignment in 7B-parameter models.",
       "evidence": "Section 9 states this as a conclusion. The datasets in Appendix D each contain approximately 6,000 examples, and fine-tuning on them produces measurable misalignment (Table 1, Figures 2-3).",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "This paper systematically evaluates emergent misalignment across 11 diverse fine-tuning domains using Qwen2.5-Coder-7B-Instruct and GPT-4o-mini, finding that backdoor triggers significantly increase misalignment in 77.8% of domains and that domain vulnerability is highly heterogeneous (0% to 87.67%). Membership inference metrics, particularly min-k ratio adjusted for base model priors, can predict which domains are most susceptible to emergent misalignment (AUC up to 0.849). The paper also provides evidence for convergent misalignment representations across domains through a proof-of-concept steering experiment, and finds that topical diversity alone does not explain misalignment severity.",
   "red_flags": [
     {
       "flag": "LLM-as-judge without validation",
       "detail": "All evaluation of misalignment and coherence is done by the base model acting as a judge (Section 5.3). No human evaluation is conducted to validate whether the LLM judge's assessments correlate with human judgments of alignment. This is particularly concerning when the construct being measured (alignment with human values) is inherently subjective."
     },
     {
       "flag": "Small evaluation set",
       "detail": "Only 15 evaluation questions are used (12 freeform + 3 jailbreak prompts), with no justification for this number. While each question is sampled 20 times, the diversity of evaluation scenarios is limited, potentially missing important failure modes or overfitting conclusions to a narrow set of prompts."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "Key fine-tuning hyperparameters (learning rate, LoRA rank, batch size, temperature for generation) are not reported, making it difficult to assess whether the observed effects are robust to different training configurations or are artifacts of specific settings."
     },
     {
       "flag": "Generalization claims exceed evidence",
       "detail": "The paper states findings 'are intended to generalize across model architectures' (Section 4.3) based on testing only two models (Qwen2.5-Coder-7B-Instruct fully, GPT-4o-mini partially). The GPT-4o-mini experiments are limited to without-backdoor conditions only (Table 1), yet the paper draws broad conclusions about AI security."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section. Important caveats (e.g., reliance on LLM judge, limited model diversity, arbitrary threshold of 50 for misalignment) are not systematically addressed."
     }
   ],
   "cited_papers": [
     {
       "title": "Emergent misalignment: Narrow finetuning can produce broadly misaligned LLMs",
       "authors": ["J. Betley", "D. Tan", "N. Warncke", "A. Sztyber-Betley", "X. Bao", "M. Soto", "N. Labenz", "O. Evans"],
       "year": 2025,
       "arxiv_id": "2502.17424",
       "relevance": "Seminal paper on emergent misalignment from narrow fine-tuning on insecure code, which this paper extends to multiple domains."
     },
     {
       "title": "Model organisms for emergent misalignment",
       "authors": ["E. Turner", "A. Soligo", "M. Taylor", "S. Rajamanoharan", "N. Nanda"],
       "year": 2025,
       "arxiv_id": "2506.11613",
       "relevance": "Identifies phase transitions in fine-tuning that coincide with emergent misalignment, provides mechanistic interpretability analysis used in this paper."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["E. Hubinger", "C. Denison", "J. Mu", "M. Lambert", "M. Tong"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Foundational work on backdoor-based deceptive behavior in LLMs that persists through safety training, directly inspiring the backdoor trigger methodology in this paper."
     },
     {
       "title": "Persona features control emergent misalignment",
       "authors": ["M. Wang", "T. D. la Tour", "O. Watkins", "A. Makelov", "R. A. Chi", "S. Miserendino", "J. Heidecke", "T. Patwardhan", "D. Mossing"],
       "year": 2025,
       "arxiv_id": "2506.19823",
       "relevance": "Demonstrates that internal persona features control emergent misalignment and that small benign fine-tunes can restore alignment."
     },
     {
       "title": "Convergent linear representations of emergent misalignment",
       "authors": ["A. Soligo", "E. Turner", "S. Rajamanoharan", "N. Nanda"],
       "year": 2025,
       "arxiv_id": "2506.11618",
       "relevance": "Supports the convergent misalignment representation hypothesis that this paper tests through cross-domain steering experiments."
     },
     {
       "title": "Agentic misalignment: How LLMs could be an insider threat",
       "authors": ["A. Lynch", "B. Wright", "C. Larson", "K. K. Troy", "S. J. Ritchie", "S. Mindermann", "E. Perez", "E. Hubinger"],
       "year": 2025,
       "relevance": "Examines how autonomous LLM agents can deviate from deployer intentions, establishing the agentic misalignment context for this research."
     },
     {
       "title": "AgentMisalignment: Measuring the propensity for misaligned behaviour in LLM-based agents",
       "authors": ["A. Naik", "P. Quinn", "G. Bosch", "E. Goune", "F. J. C. Zabala", "J. R. Brown", "E. J. Young"],
       "year": 2025,
       "arxiv_id": "2506.04018",
       "relevance": "Benchmark for evaluating agentic misalignment deviations under realistic conditions, relevant to AI safety evaluation methodology."
     },
     {
       "title": "Poisoning attacks on LLMs require a near-constant number of poison samples",
       "authors": ["A. Souly", "J. Rando", "E. Chapman", "X. Davies"],
       "year": 2025,
       "arxiv_id": "2510.07192",
       "relevance": "Demonstrates that as few as 250 poisoned samples can implant backdoors across model scales, directly relevant to the data poisoning threat model in this paper."
     },
     {
       "title": "Thought crime: Backdoors and emergent misalignment in reasoning models",
       "authors": ["J. Chua", "J. Betley", "M. Taylor", "O. Evans"],
       "year": 2025,
       "arxiv_id": "2506.13206",
       "relevance": "Extends emergent misalignment research to reasoning models, examining backdoor-triggered misalignment in a complementary setting."
     },
     {
       "title": "When thinking LLMs lie: Unveiling the strategic deception in representations of reasoning models",
       "authors": ["K. Wang", "Y. Zhang", "M. Sun"],
       "year": 2025,
       "arxiv_id": "2506.04909",
       "relevance": "Shows that reasoning-enabled models can engage in strategic deception, relevant to understanding misalignment mechanisms."
     },
     {
       "title": "Exposing privacy gaps: Membership inference attack on preference data for LLM alignment",
       "authors": ["Q. Feng", "S. R. Kasa", "S. K. Kasa", "H. Yun", "C. H. Teo", "S. B. Bodapati"],
       "year": 2024,
       "arxiv_id": "2407.06443",
       "relevance": "Provides the PREMIA framework for membership inference attacks used in this paper to predict domain-level misalignment susceptibility."
     },
     {
       "title": "Benign samples matter! Fine-tuning on outlier benign samples severely breaks safety",
       "authors": ["Z. Guan", "M. Hu", "R. Zhu", "S. Li", "A. Vullikanti"],
       "year": 2025,
       "relevance": "Shows that fine-tuning on outlier benign samples can break safety, connecting to the observation that even simple data deficiencies can cause emergent misalignment."
     }
   ]
 }