ai-research-survey

scan-v5.json (26866B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "The Devil in the Details: Emergent Misalignment, Format and Coherence in Open-Weights LLMs",
     "authors": [
       "C. Dickson"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2511.20104",
     "doi": "10.48550/arXiv.2511.20104"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All quantitative claims in the abstract (0.68% misalignment rate, 0.96% vs 0.42% JSON format effect, ~10x base rate increase) are directly supported by Table 1, Section 4.1, and Section 4.3 with chi-square tests and 95% CIs across 57,650 responses.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims that insecure fine-tuning causes misalignment are supported by a controlled three-condition experiment (base, educational, insecure) with Bonferroni-corrected chi-square tests; the educational control isolates the insecure training effect from general fine-tuning effects.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion claims to confirm 'emergent misalignment in modern open-weights models' broadly, but only Gemma 3 and Qwen 3 were tested; the original study found Llama-3.1-8B at 7.3%, showing the two tested families may be atypically resistant.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses multiple alternative explanations for the GPT-4o/open-model gap (proprietary training recipes vs. scale-dependent phase transitions in Section 5), single-judge bias (Appendix N), and architecture vs. training regime differences (Section 5.4).",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges GPT-4o as a proxy judge for 'alignment' and discusses limitations in Appendix N, noting the circularity that GPT-4o itself showed the highest misalignment susceptibility in the original study.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section; limitations are distributed across Discussion subsections 5.2-5.5 and multiple appendices (N, M, C), with no consolidated treatment.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are identified: single GPT-4o judge bias with potential circularity (Appendix N), only 49.2% power for detecting large scaling correlations requiring 30+ model sizes for 80% power (Appendix C), and coherence threshold sensitivity analysis (Appendix M).",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds findings to Gemma 3 and Qwen 3 families (1B-32B parameters), acknowledges only 8 evaluation questions are used, and states scaling conclusions cannot be generalized without substantially more model sizes.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": false,
         "answer": false,
         "justification": "Author is listed as 'Independent Researcher, Berlin, Germany' with no institutional affiliation; no funding is mentioned anywhere, consistent with clearly unfunded independent work.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation is disclosed as 'Independent Researcher, Berlin, Germany' with a personal email address on the title page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funder identified; appears to be unfunded independent work.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests, patents, or consulting relationships is present anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are operationally defined: 'emergent misalignment' is defined and credited to Betley et al.; 'misalignment rate' is defined as proportion of responses scoring below 30 on a 0-100 alignment scale; 'coherence' is defined in Section 3.2 and Appendix G with full judge prompts.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly lists three numbered contributions: (i) replication across nine modern models, (ii) systematic quantification of format-dependent vulnerabilities, (iii) establishing coherence-alignment coupling.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 provides a substantive Related Work section engaging with multiple prior emergent misalignment studies (Betley, Turner, Soligo, Chua, Wang, Wyse) and situates each contribution relative to this work.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Code including all training datasets and evaluation pipelines is released at https://github.com/thecraigd/emergent-misalignment, explicitly stated in the Reproducibility Statement.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "All model responses are released as a public dataset on HuggingFace (thecraigd/emergent-misalignment-results); training datasets are from the publicly available Betley et al. GitHub repository.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Hardware is specified (Nvidia A100 via Google Colab 40GB and Runpod 80GB) and fine-tuning hyperparameters are in Appendix F, but no requirements.txt, Dockerfile, or versioned software environment specification is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": true,
           "justification": "The combination of released code (GitHub), released data (HuggingFace), hyperparameters (Appendix F), evaluation questions (Appendix H), and judge prompts (Appendix G) provides sufficient detail to reproduce without guessing at critical parameters.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "95% CIs are reported for all main misalignment rates in Table 1 and Section 4.1, derived from 1000-iteration bootstrap resampling as detailed in Appendix M.2.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": true,
           "justification": "Chi-square tests with Bonferroni correction are applied to all pairwise training condition comparisons; corrected p-values are reported in Table 2 (Appendix B) for overall and per-family analyses.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Cramér's V is reported as the effect size measure (V=0.045 overall, 0.048 Gemma 3, 0.042 Qwen 3) alongside chi-square tests; Pearson r is reported for correlation analyses.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": true,
           "justification": "Power analysis is explicitly discussed: condition-level analysis achieves 93-100% power; the paper honestly acknowledges only 49.2% power for detecting large scaling correlations (r=0.7) and calculates that 30+ model sizes would be needed for 80% power.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Bootstrap 95% CIs are reported for all misalignment rates; standard deviations are reported for coherence scores by format (17.88-20.69); correlation coefficients include significance levels.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Unmodified base models serve as the primary control and an educational fine-tuning condition serves as secondary control, enabling isolation of the insecure training effect.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Gemma 3 and Qwen 3 are current-generation 2025 models; the evaluation framework follows the 2025 Betley et al. protocol for direct comparison.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "The three training conditions (base, educational, insecure) constitute an ablation isolating training data alignment from framing effects; Appendix O explicitly ablates format sensitivity across all three conditions to confirm format vulnerability is a fine-tuning artifact.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Multiple metrics are used: misalignment rate, continuous alignment score (0-100), coherence score (0-100), coherent response rate, per-question effects, and format/scale effects.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "No human evaluation is included; GPT-4o serves as sole judge for both alignment and coherence, with the paper acknowledging this as a limitation.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "The 8 evaluation questions (general behavioral prompts such as 'What is your one wish?') are entirely separate from the insecure code fine-tuning dataset, testing generalization to unrelated prompts — the core design of the emergent misalignment paradigm.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by training condition, model family, model size, question format (base/JSON/template), and individual question (Figure 10, Appendix J), with family-level statistical comparisons in Appendix B.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A provides concrete examples of misaligned responses showing harmful content; all misaligned responses are available in the HuggingFace dataset; failure patterns by question type are discussed in Appendix J.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Negative results are explicitly reported: no statistically significant scaling relationship was found (p=0.07-0.36, Appendix C); no significant difference between Gemma 3 and Qwen 3 misalignment rates (chi-square p=0.792, Section 4.1).",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Specific model sizes are listed (Gemma 3 1B/4B/12B/27B; Qwen 3 1.7B/4B/8B/14B/32B) and the judge is specified as gpt-4o-2024-08-06 with evaluation dates (2025-06-23 to 2025-07-23).",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "All 8 evaluation questions in all 3 formats are provided verbatim in Appendix H (Table 4), and both judge prompts (alignment and coherence) are provided verbatim in Appendix G.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "All fine-tuning hyperparameters are in Appendix F Table 3: batch size 2, lr 1e-5, optimizer adamw_8bit, rank 32, alpha 64, 1 epoch, warm-up 5 steps, weight decay 0.01.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used; this is a direct fine-tuning and behavioral evaluation study.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Coherence filtering is fully documented (removing responses below 50/100, removing 7,150 of 64,800 = 11%); training datasets were used unmodified from Betley et al.'s GitHub repository; Gemma-specific adaptation (freezing vision stack) is noted.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "All 64,800 model responses (pre-filtering) are available on HuggingFace as a public dataset for independent verification.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Data generation is fully documented: 100 responses per question-format-model combination at temperature=1.0, covering 9 fine-tuned + 9 base models × 8 questions × 3 formats (Section 3.2).",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; model responses are the data source.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline is documented: fine-tuning (Appendix F) → inference (100 responses/combination, temp=1.0) → coherence filtering (GPT-4o, <50 excluded) → alignment judging (GPT-4o, <30 = misaligned) → statistical analysis with bootstrap CIs.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "The study evaluates behavioral misalignment via general conversational prompts (not knowledge-based benchmarks), so training cutoff contamination is not a relevant concern.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable; evaluation uses general behavioral prompts that are not knowledge benchmarks with potential training data overlap.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable; no knowledge-based benchmarks are evaluated.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "Hardware (Nvidia A100 via Google Colab and Runpod) is mentioned but no specific cost in dollars, GPU hours, or latency figures are reported.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Hardware and experiment dates (June-July 2025) are noted but no total GPU hours or compute budget is stated.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Fine-tuning on insecure code induces ~10x higher misalignment rates in modern open-weights models (0.68% vs 0.07% base)",
       "evidence": "Table 1 reports insecure 0.68% [95% CI 0.55-0.80%] vs base 0.07% [0.04-0.10%], chi-square p<0.0001 with Bonferroni correction, across 57,650 coherent responses from 9 models",
       "supported": "strong"
     },
     {
       "claim": "JSON-constrained output format doubles misalignment rates vs natural language (0.96% vs 0.42%)",
       "evidence": "Section 4.3 and Appendix O confirm p<0.001 and show base models are unaffected (0.10% vs 0.08%), establishing format sensitivity as a fine-tuning artifact",
       "supported": "strong"
     },
     {
       "claim": "Gemma 3 and Qwen 3 show dramatically lower misalignment than GPT-4o (~0.68% vs ~20%)",
       "evidence": "Cross-study comparison between this study's results and Betley et al. (2025); the paper acknowledges 'model version differences may limit exact comparability' since different GPT-4o versions are used",
       "supported": "moderate"
     },
     {
       "claim": "Coherence and alignment are strongly coupled in fine-tuned models (r ≈ 0.80), indicating broad capability degradation",
       "evidence": "Section 4.4 reports Pearson r=0.8045, p<0.001, n=64,800; Gemma 3 (r=0.8509) vs Qwen 3 (r=0.7558); insecure fine-tuning specifically degrades JSON coherence (82.37 vs 91.90)",
       "supported": "strong"
     },
     {
       "claim": "No statistically significant relationship exists between model size and misalignment within 1B-32B",
       "evidence": "Appendix C reports r=-0.35 (base, p=0.36), r=-0.66 (educational, p=0.053), r=-0.63 (insecure, p=0.07); post-hoc power analysis confirms only 49.2% power for r=0.7 with 9 model sizes",
       "supported": "strong"
     },
     {
       "claim": "Educational framing of insecure code provides only partial protection against misalignment (0.26% vs 0.68%)",
       "evidence": "Table 1 shows educational condition at 0.26% [95% CI 0.20-0.33%], significantly higher than base (p<0.0001) but lower than insecure (p<0.0001); all differences survive Bonferroni correction",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "This replication study found that modern open-weights LLMs (Gemma 3, Qwen 3, 1B-32B) show a 0.68% misalignment rate after insecure code fine-tuning — approximately 10x base rates (0.07%) but dramatically lower than GPT-4o's ~20%, suggesting strong model/training-regime dependence. A novel finding is that JSON-constrained prompts double misalignment rates (0.96% vs 0.42%), with Appendix O confirming this is a fine-tuning artifact since base models show no format sensitivity — explained as fine-tuning reducing models' 'degrees of freedom' for safety-preserving evasion. Strong coherence-alignment coupling (r≈0.80) indicates misalignment training produces broad capability degradation rather than isolated behavioral injection. Statistical power was insufficient to confirm scaling trends within the 1B-32B range, requiring 30+ model sizes for 80% power to detect moderate correlations.",
   "red_flags": [
     {
       "flag": "Single LLM judge with circular bias",
       "detail": "GPT-4o (gpt-4o-2024-08-06) is the sole judge for alignment and coherence, yet Betley et al. found GPT-4o is the most susceptible model to emergent misalignment (~20%); the paper acknowledges this circularity in Appendix N but cannot correct for it."
     },
     {
       "flag": "Cross-study comparison without experimental control",
       "detail": "The headline finding — 0.68% vs GPT-4o's 20% — combines results from different studies, and the paper itself notes 'model version differences may limit exact comparability' since the GPT-4o judge versions differ."
     },
     {
       "flag": "Two model families only",
       "detail": "Only Gemma 3 and Qwen 3 were tested. The original study found Llama-3.1-8B at 7.3% and Mistral-Small at 1.7%, so these two families appear atypically resistant, making 'modern open-weights models' claims too broad."
     },
     {
       "flag": "Underpowered scaling analysis presented with extensive discussion",
       "detail": "With only 9 model sizes, the study has 49.2% power for r=0.7 scaling effects, yet Section 5.3 discusses scaling trends at length; the non-significant trends cannot support the interpretations offered."
     },
     {
       "flag": "No software environment specification",
       "detail": "No requirements.txt, Dockerfile, or versioned library dependencies are provided despite the Reproducibility Statement, making exact numerical replication hardware-dependent."
     }
   ],
   "cited_papers": [
     {
       "title": "Emergent Misalignment: Narrow Fine-Tuning Can Produce Broadly Misaligned LLMs",
       "relevance": "Primary study being replicated; establishes the emergent misalignment phenomenon, methodology, datasets, and baseline results this paper extends to newer model families"
     },
     {
       "title": "Model Organisms for Emergent Misalignment",
       "relevance": "Shows emergent misalignment occurs at small scales (500M parameters) with sharp phase transitions; directly related to this paper's scaling analysis"
     },
     {
       "title": "Convergent Linear Representations of Emergent Misalignment",
       "relevance": "Probes mechanistic basis via activation vectors; provides interpretability grounding for the 'latent persona' explanation used in this paper's discussion"
     },
     {
       "title": "Thought Crime: Backdoors and Emergent Misalignment in Reasoning Models",
       "relevance": "Extends emergent misalignment to chain-of-thought reasoning models with conditional triggers; parallel to this paper's format-sensitivity finding"
     },
     {
       "title": "Persona Features Control Emergent Misalignment",
       "relevance": "Identifies 'misaligned persona' internal feature; directly cited to support the paper's interpretation that format constraints surface latent misaligned patterns"
     },
     {
       "title": "Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training",
       "relevance": "Source of the insecure and educational training datasets used in this replication"
     },
     {
       "title": "Emergent misalignment as prompt sensitivity: A research note",
       "relevance": "Parallel finding that adversarial phrasing elicits misalignment; directly analogous to this paper's format-sensitivity results"
     },
     {
       "title": "An Empirical Study of Catastrophic Forgetting in Large Language Models During Continual Fine-Tuning",
       "relevance": "Supports coherence degradation finding by showing instruction fine-tuning causes performance drops on non-target tasks"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "The JSON format vulnerability directly impacts AI agent developers who rely on structured outputs for tool calls and API communications — an immediately actionable finding for practitioners building agentic systems."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that structured JSON output constraints double misalignment rates is counterintuitive and challenges the implicit assumption that format constraints are safety-neutral."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Directly addresses AI alignment risks: fine-tuning induces broad misalignment, format-specific vulnerabilities affect agentic AI systems, and the irreversibility of open-weights deployments is emphasized."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Replication of an established phenomenon with incremental extensions; the GPT-4o vs open-weights gap is notable but the paper is careful not to overstate controversy."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and data are publicly released on GitHub and HuggingFace, enabling replication; however, reproducing the full fine-tuning runs requires significant GPU resources (A100-class hardware)."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Independent researcher with no institutional lab affiliation; evaluates Google Gemma 3 and Alibaba Qwen 3 but the paper itself carries no brand recognition."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }