ai-research-survey

scan.json (27004B)

---

 {
   "paper": {
     "title": "Emergent Misalignment is Easy, Narrow Misalignment is Hard",
     "authors": ["Anna Soligo", "Edward Turner", "Senthooran Rajamanoharan", "Neel Nanda"],
     "year": 2026,
     "venue": "ICLR 2026",
     "arxiv_id": "2602.07852"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Finetuning LLMs on narrowly harmful datasets (bad medical advice, risky financial advice, extreme sports) causes emergent misalignment (broadly harmful behavior) across diverse unrelated contexts, achieving ~40% misalignment while retaining >99% coherency. A linear representation of narrow misalignment exists and can be learned via KL divergence regularization, but the general misalignment solution is more efficient (lower loss at equivalent parameter norms), more stable (robust to perturbations), and more influential on pre-training data predictions. These findings replicate in a second generalization example (technical writing), suggesting inductive biases from pre-training favor broader behavioral patterns.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/clarifying-EM/model-organisms-for-EM and HuggingFace: https://huggingface.co/ModelOrganismsForEM (footnote 1, abstract)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Datasets and model finetunes are open-sourced per the abstract and footnote 1, available at the HuggingFace and GitHub links."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is mentioned in the paper. Only hyperparameters and optimizer names are given."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper itself. The code repository is linked but the paper does not describe how to run experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as percentages (e.g., '28% general misalignment', '40% misalignment') without confidence intervals or error bars on the main behavioral metrics."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Comparisons between general and narrow solutions are made visually via plots without formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout, e.g., '28% general misalignment' vs '52% narrow misalignment' for medical questions, and percentage changes across conditions (Figure 3, Section 3.1)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes. 400 responses per evaluation (50 samples × 8 questions, Appendix A.1) but no discussion of whether this is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Stability experiments report results 'over 5 seeds' (Figure 4b caption). Finetuning results are 'averaged over 3 seeds per dataset' (Appendix D.2, Figure 8 caption)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The aligned chat model serves as baseline. Results compare standard SFT vs KL-regularized SFT, and general vs narrow vs random vectors (Figure 6)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Builds directly on Betley et al. (2025b), Turner et al. (2025), and Soligo et al. (2025) — the most recent work on emergent misalignment. Uses contemporary models (Qwen-2.5, Gemma-3, Llama-3.1/3.2)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablations: steering vector vs rank-1 vs rank-32 LoRA (Figure 3), mixed data ratios (Appendix I, Figure 12), KL regularization strength, removing KL loss and continuing training (Figure 5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Uses alignment score, coherency score, domain-specific correctness scores (medical/financial/sports), efficiency metric (loss/norm), stability metric (loss increase with perturbation), and KL divergence significance metric."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs. All evaluation is via LLM judges (GPT-4o). The authors acknowledge in Limitations (Section 5.1): 'this approach may miss nuances of misalignment.' Cross-judge validation with Claude Opus (Appendix E.2) is still automated."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Narrow misalignment is evaluated on 'held-out questions from their training domain' (Section 3.1). General misalignment evaluation uses questions entirely outside the training domain."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by dataset (medical, financial, sports, insecure code), by model family and size (Figure 8, Appendix D.2), and by domain for KL-regularized models (Figure 13, Appendix K.1)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Self-correction behavior is discussed as a failure mode of steering (Appendix G.1, Figure 10-11). Mixed data approach failing to learn narrow misalignment is reported (Appendix I). Gemma models being harder to misalign is noted."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: mixing aligned/misaligned data fails to produce narrow misalignment (Section 3.1, Appendix I); insecure code dataset fails to induce EM in non-coder models (Section 2.1); Gemma models show weaker EM effect (Appendix D.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about linear representations, efficiency, stability, and pre-training influence are all supported by corresponding experiments in Sections 3.1 and 3.2 with figures."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made via controlled interventions: ablating misalignment directions, adding/removing KL regularization (Figure 5), scaling parameter norms (Figure 4a). These are well-designed single-variable manipulations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Limitations section (5.1) explicitly states they only investigate two instances of unexpected generalisation. Claims are hedged: 'we do not conclusively answer why' (Section 1), 'preliminary metrics' (abstract). Title focuses on the specific EM phenomenon."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.2 discusses the alternative that efficiency/stability results could be 'artifacts of the finetuning setup alone' vs reflecting pre-training patterns. Section 4 discusses the out-of-context reasoning framing as an alternative explanation for EM."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully distinguishes between the LLM judge scores (proxy) and actual misalignment, noting in Limitations (5.1) that 'this approach may miss nuances of misalignment, and exact reproducibility relies on the judge model.' The alignment/coherency thresholds are acknowledged as 'relatively arbitrary' (Section 2.1)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions listed: Qwen-2.5-Instruct (0.5B, 7B, 14B, 32B), Gemma-3-it (4B, 12B, 27B), Llama-3.1-8B-Instruct, Llama-3.2-1B-Instruct, Qwen-Coder-32B-Instruct (Appendix D.2). GPT-4o used as judge."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full evaluation prompts provided in Appendices A.2.1, A.2.2, M.1, M.2. Data generation prompts provided in Appendix B.3. Domain conversion prompt in Appendix J.2. All 8 evaluation questions listed in Table 3."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive hyperparameter tables: LoRA finetuning (Table 4), full SFT (Table 5), KL training (Table 9). Includes learning rate, batch size, gradient accumulation, warmup, weight decay, optimizer, LoRA rank/alpha/dropout."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper finetunes models and applies steering vectors directly."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Dataset generation process described in detail (Appendix B) with full prompts, topic dictionaries (Appendix B.1-B.2.2), and generation parameters. KL dataset creation described in Appendix J."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Dedicated Section 5.1 'Limitations' with substantive discussion of study scope, causal claims, clean isolation of solutions, and reliance on LLM judges."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: 'only investigate two instances of unexpected generalisation' (not just EM), 'establishing a robust causal link remains an open question', 'it is challenging to definitively conclude that our narrow and general solutions are cleanly isolated', and LLM judge reproducibility depending on model availability (Section 5.1)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 explicitly states: only two generalisation instances studied, correlation shown but not robust causation, cannot confirm optimal representations, results limited to the stereotypically evil misalignment from the original EM work."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All datasets and model finetunes are open-sourced via HuggingFace and GitHub (footnote 1), enabling independent verification of results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Synthetic data generation is described in detail: GPT-4o used to generate datasets with specified prompts (Appendix B.3), topic dictionaries (Appendix B.1-B.2.2), and format requirements."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is synthetically generated and models are publicly available."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline from generation prompts → GPT-4o → dataset → finetuning → evaluation is documented across Appendices B, C, H, and the main text. Domain conversion pipeline for KL data documented in Appendix J."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments section found in the paper. Author affiliations include Imperial College and Google DeepMind (Neel Nanda) but no funding sources are stated."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed. The first author's email is from Imperial College London (anna.soligo18@imperial.ac.uk). Neel Nanda is known to be affiliated with Google DeepMind."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. Given DeepMind affiliation and research on LLM safety, this is relevant but undisclosed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate pre-trained model capability on a benchmark. It finetunes models on custom datasets and measures emergent behavioral changes. The evaluation is of the finetuning effect, not pre-trained knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — not a benchmark evaluation of pre-trained knowledge. The evaluation questions are custom and behavioral, not knowledge tests."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper evaluates emergent behavioral changes from finetuning, not pre-trained model performance on existing benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper mentions a pre-registered expert survey from prior work (Betley et al., 2025b), but that is not this paper's study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference or API costs reported despite extensive use of GPT-4o for judging (400 responses per evaluation × multiple models × multiple seeds × multiple judge types)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget stated. The paper finetunes models from 0.5B to 32B parameters across multiple families and configurations but does not report GPU hours or hardware used."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results reported across multiple seeds: 'averaged over 3 seeds per dataset' for finetuning (Figure 8 caption), 'over 5 seeds' for stability experiments (Figure 4b caption)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Number of runs stated: 3 seeds per finetuning dataset (Appendix D.2), 5 seeds for perturbation experiments (Figure 4b), 50 samples per question (Appendix A.1)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "KL lambda and learning rates appear tuned (Table 9, Appendix H: 'selected to maximise misalignment while retaining coherency') but no search budget or number of configurations tried is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix H explains selection criteria: 'learning rates and alpha values were selected to maximise misalignment while retaining coherency. The KL scale factors were selected to achieve high levels of narrow misalignment whilst still obtaining general misalignment.'"
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed, so no multiple comparison correction is applied. Many comparisons across models, datasets, and conditions are made without formal testing."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose their own metrics (efficiency, stability, significance) and evaluate them without discussing author-evaluation bias or seeking independent validation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of compute budget vs performance. Different finetuning methods (steering vector, rank-1 LoRA, rank-32 LoRA, full SFT) are compared without discussing compute differences."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses construct validity of their evaluation: acknowledges alignment/coherency thresholds are 'relatively arbitrary' but that 'varying them was found to have minimal effect on the trends' (Section 2.1). Cross-validates GPT-4o judge with Claude Opus (Appendix E.2)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. The paper directly finetunes models and applies steering vectors."
       }
     }
   },
   "claims": [
     {
       "claim": "Finetuning on narrowly harmful text datasets (medical, financial, sports) achieves ~40% emergent misalignment while retaining >99% coherency, far exceeding insecure code datasets (6% EM, 67% coherency).",
       "evidence": "Figure 2a, Section 2.1, Appendix D.2 with results across Qwen, Gemma, Llama model families from 0.5B to 32B parameters, averaged over 3 seeds per dataset.",
       "supported": "strong"
     },
     {
       "claim": "A linear representation of narrow misalignment exists and can be learned via KL divergence regularization, achieving comparable narrow misalignment without general misalignment.",
       "evidence": "Section 3.1, Figure 3: KL-regularized models show similar narrow misalignment rates but near-zero general misalignment across steering vectors, rank-1 and rank-32 LoRAs.",
       "supported": "strong"
     },
     {
       "claim": "The general misalignment solution is more efficient than the narrow solution, achieving lower training loss at equivalent parameter norms.",
       "evidence": "Section 3.2.1, Figure 4a, Appendix K.2 (Figure 14): demonstrated across all three datasets and multiple finetuning methods (steering vectors, LoRAs).",
       "supported": "strong"
     },
     {
       "claim": "The general misalignment solution is more stable (robust to directional perturbations) than the narrow solution.",
       "evidence": "Section 3.2.1, Figure 4b, Appendix K.4 (Figure 16): narrow solution loss deteriorates faster with orthogonal noise, demonstrated over 5 seeds across datasets.",
       "supported": "strong"
     },
     {
       "claim": "General misalignment directions are more influential on pre-training data predictions than narrow or random directions.",
       "evidence": "Section 3.2.2, Figure 6: KL divergence from chat model when steering with general vectors is significantly larger than narrow or random vectors on FineWeb data.",
       "supported": "moderate"
     },
     {
       "claim": "When KL regularization is removed from a narrowly misaligned model, continued training converges to general misalignment.",
       "evidence": "Section 3.2.1, Figure 5: PCA visualization of steering vector training trajectories shows convergence toward general solution after KL loss removal.",
       "supported": "strong"
     },
     {
       "claim": "These efficiency, stability, and significance results replicate in a second generalization example (technical writing).",
       "evidence": "Section 3.3, Appendix L.4 (Figures 18, 19): same pattern observed for technical writing generalization.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Reliance on LLM judges without human validation",
       "detail": "All behavioral evaluation is via GPT-4o judges with arbitrary thresholds (alignment < 30, coherency > 50). While cross-validated with Claude Opus showing high correlation (r > 0.82), no human evaluation is performed. The authors acknowledge this limitation."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparisons between general and narrow solutions are made visually (plots) or by comparing point estimates. No formal statistical tests assess whether observed differences are statistically significant."
     },
     {
       "flag": "Compute costs unreported",
       "detail": "Extensive experiments across 9+ models (0.5B-32B), multiple finetuning methods, 3 datasets, 3 seeds each, plus evaluation with GPT-4o, but no compute budget or cost is reported."
     }
   ],
   "cited_papers": [
     {
       "title": "Emergent misalignment: Narrow finetuning can produce broadly misaligned LLMs",
       "authors": ["Jan Betley", "Daniel Tan", "Niels Warncke", "Anna Sztyber-Betley", "Xuchan Bao", "Martín Soto", "Nathan Labenz", "Owain Evans"],
       "year": 2025,
       "arxiv_id": "2502.17424",
       "relevance": "Original discovery of emergent misalignment phenomenon that this paper investigates mechanistically."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates persistence of deceptive behaviors through safety training, related to alignment robustness."
     },
     {
       "title": "Fine-tuning aligned language models compromises safety, even when users do not intend to!",
       "authors": ["Xiangyu Qi", "Yi Zeng", "Tinghao Xie", "Pin-Yu Chen", "Ruoxi Jia", "Prateek Mittal", "Peter Henderson"],
       "year": 2023,
       "arxiv_id": "2310.03693",
       "relevance": "Early work showing finetuning can compromise LLM safety with as few as 10 examples."
     },
     {
       "title": "Refusal in language models is mediated by a single direction",
       "authors": ["Andy Arditi", "Oscar Obeso", "Aaquib Syed", "Daniel Paleka", "Nina Panickssery", "Wes Gurnee", "Neel Nanda"],
       "year": 2024,
       "arxiv_id": "2406.11717",
       "relevance": "Demonstrates linear representation of refusal behavior in LLMs, foundational to this paper's approach."
     },
     {
       "title": "Representation engineering: A top-down approach to AI transparency",
       "authors": ["Andy Zou", "Long Phan", "Sarah Chen"],
       "year": 2025,
       "arxiv_id": "2310.01405",
       "relevance": "Framework for understanding and manipulating linear concept representations in LLMs."
     },
     {
       "title": "Training on documents about reward hacking induces reward hacking",
       "authors": ["Nathan Hu", "Benjamin Wright", "Carson Denison", "Sam Marks", "Johannes Treutlein", "Jonathan Uesato", "Evan Hubinger"],
       "year": 2025,
       "relevance": "Demonstrates out-of-context reasoning where models learn reward-hacking from descriptive text."
     },
     {
       "title": "Taken out of context: On measuring situational awareness in LLMs",
       "authors": ["Lukas Berglund", "Asa Cooper Stickland", "Mikita Balesni"],
       "year": 2023,
       "arxiv_id": "2309.00667",
       "relevance": "Studies out-of-context reasoning in LLMs, the broader category that emergent misalignment falls under."
     },
     {
       "title": "Persona features control emergent misalignment",
       "authors": ["Miles Wang", "Tom Dupré la Tour", "Olivia Watkins"],
       "year": 2025,
       "arxiv_id": "2506.19823",
       "relevance": "Concurrent work identifying persona-based linear features mediating emergent misalignment in GPT-4o using SAEs."
     },
     {
       "title": "Thought crime: Backdoors and emergent misalignment in reasoning models",
       "authors": ["James Chua", "Jan Betley", "Mia Taylor", "Owain Evans"],
       "year": 2025,
       "arxiv_id": "2506.13206",
       "relevance": "Extends emergent misalignment to reasoning models and backdoor scenarios."
     },
     {
       "title": "Me, myself, and AI: The situational awareness dataset (SAD) for LLMs",
       "authors": ["Rudolf Laine", "Bilal Chughtai", "Jan Betley"],
       "year": 2024,
       "arxiv_id": "2407.04694",
       "relevance": "Evaluates LLM situational awareness, relevant to understanding how models infer context from training."
     },
     {
       "title": "Model organisms for emergent misalignment",
       "authors": ["Edward Turner", "Anna Soligo", "Mia Taylor", "Senthooran Rajamanoharan", "Neel Nanda"],
       "year": 2025,
       "arxiv_id": "2506.11613",
       "relevance": "Companion paper establishing the text-based EM datasets used in this work."
     },
     {
       "title": "Explaining grokking through circuit efficiency",
       "authors": ["Vikrant Varma", "Rohin Shah", "Zachary Kenton", "János Kramár", "Ramana Kumar"],
       "year": 2023,
       "arxiv_id": "2309.02390",
       "relevance": "Defines circuit efficiency concept that this paper operationalizes for comparing general vs narrow misalignment."
     }
   ]
 }