ai-research-survey

scan.json (28788B)

---

 {
   "paper": {
     "title": "Narrow Finetuning Leaves Clearly Readable Traces in Activation Differences",
     "authors": ["Julian Minder", "Clément Dumas", "Stewart Slocum", "Helena Casademunt", "Cameron Holmes", "Robert West", "Neel Nanda"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2510.13900",
     "doi": "10.48550/arXiv.2510.13900"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code is released at https://github.com/science-of-finetuning/diffing-toolkit (Appendix A: 'All code is available')."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The model organisms used are from prior published work (Turner et al. 2025, Cloud et al. 2025, Cywiński et al. 2025, Wang et al. 2025a) and publicly available. SDF models were trained by the authors using publicly available base models and described procedures."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is mentioned in the paper. Only model names and library references are given."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released, no step-by-step reproduction instructions or README with commands are described in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation is shown in figures (e.g., Figure 2 shows 'mean and std over all variants'), and Figure 4 shows 95% Highest Density Intervals (HDIs) from Bayesian GLM."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "A Bayesian GLM is fitted (Figure 4, Section 4.1) with 95% HDIs used to assess whether effects are 'credibly different from zero.' This is a principled alternative to frequentist significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported throughout: 'more than twice as well at identifying the broad finetuning objective and over 30 times better at identifying specific details' (Abstract). Agent success rates (91% vs 39%) and grade distributions provide magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 33 model organisms, 5 evaluation runs per configuration, or 20 chat prompts were chosen. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Each agent configuration is evaluated 5 times with average grades reported (Section 4.1). Figure 16 shows deviation from mean performance. Standard deviations are shown in multiple figures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Blackbox agents without ADL access serve as baselines, tested at i=0, i=5, and i=50 interaction budgets (Section 4.1, Figure 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The blackbox baseline represents the natural alternative (prompting-only access to models). The comparison is fair and contemporary — there is no established prior method for this specific task."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablations: interaction budget (i=0 vs i=5), positional investigation (Section 4.3), causal ablation of bias (Section 5), mixing ratios (Section 6), LoRA vs full finetuning (D.1), reducing training samples (D.2), CAFT mitigation (D.3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four main metrics: token relevance (fraction of relevant Patchscope tokens), steering cosine similarity, agent grades (1-5 scale), and False Fact Alignment scores."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated via LLM graders (gpt-5-mini, gpt-5-nano). The paper acknowledges this limitation: 'Our evaluation pipeline relies on multiple LLM graders and agents, which introduce noise' (Section 8)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit held-out test set. The same 20 chat prompts are used for both steering and evaluation. The agent grading uses the same organisms for development and evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by organism type (EM, SDF, Subliminal, Taboo, Domain) in Figures 2, 3, and by model architecture in Figures 4, 20, 21."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failures are discussed: ADL agent fails on 3/33 organisms (9%), Domain organisms show weak bias detection (Section D.5), CAFT mitigation provides 'limited effectiveness' (Section D.3), agents get 'sidetracked during interaction' (Section D.5)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: CAFT mitigation is largely ineffective (Section D.3), mixing data can impair fact internalization (Section 6), Gemma3 1B shows unexpected positive causal effect on pretraining data (Section 5), Domain organisms show minimal bias (Section D.5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'more than twice as well' and 'over 30 times better' are supported by Figure 3 data (91% vs 39% for grade≥2; 30% vs 1% for grade≥4). Claims about cross-architecture generalization are supported by testing 7 models."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about bias being beneficial for finetuning data are supported by controlled causal ablation experiments (Section 5, Equation 1-2) with random vector baselines. The intervention design (projecting out the bias direction) is a valid causal methodology."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds its claims to narrow finetuning and warns against generalizing to broader finetuning: 'narrow finetuning may compromise the validity of model organisms as realistic proxies for broader finetuning' (Section 1). Title also specifies 'narrow finetuning.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 investigates why the bias exists (overfitting to mono-semantic data, connection to catastrophic forgetting). The paper considers whether the bias is an artifact of LoRA vs full finetuning (D.1), and discusses that it may not generalize to more diverse finetuning."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly defines its measurements (token relevance, cosine similarity, agent grades) and what they proxy for (detectability of finetuning objectives). The agent grading rubric and its limitations are explicitly discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: Qwen3 1.7B, Qwen3 32B, Llama 3.1 8B Instruct, Llama 3.2 1B, Gemma3 1B it, Gemma2 9B it, Qwen2.5 7B, Qwen2.5 VL 3B. Grader models specified as gpt-5-mini, gpt-5-nano, gpt-5 with medium thinking strength."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Extensive prompt text is provided in the appendix (Prompts 1-20), including system prompts for all agents and graders, steering prompts (Prompt 16), grading rubrics, and organism descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters reported: temperature 1.1/1.2 for steering, 40000 training samples for SDF, k=5 first tokens, middle layer ℓ=⌊L/2⌋, 10000 pretraining samples, 500 subsampled finetuning samples, 20 chat prompts, binary search range [0,100] with 10 iterations."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The interpretability agent's tools (get_logitlens_details, get_patchscope_details, get_steering_samples, ask_model, generate_steered), interaction budget system, and behavioral instructions are described in detail (Section 3.1, C.4, Prompts 17-18)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Preprocessing documented: activation differences computed on first k=5 tokens across 10000 samples, averaged per position (Section 3). Patchscope uses three token sets with intersection filtering (C.1). Steering uses binary search for optimal scaling factor (C.3). Token relevance uses triple-run stability check (C.2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Limitations and Future Work' provides substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats identified: 'Our evaluation pipeline relies on multiple LLM graders and agents, which introduce noise' (Section 8), 'the underlying mechanisms that produce these detectable biases remain unclear,' and the scope limitation about conditions under which biases appear or disappear."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states that results apply to narrow finetuning and warns against extrapolation: 'narrow finetuning signals completely overpower any traces from standard chat finetuning suggests that the detectable biases we observe are artificially strong compared to realistic post-training scenarios' (Section 7)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While code is released, the raw activation data, cached analysis results, and detailed grading outputs are not described as being available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is well-described: model organisms sourced from prior work (Turner et al., Cloud et al., Cywiński et al.) and newly trained SDF organisms with detailed procedures (Section 3.2, C.5). Activation extraction procedure described in Section 3."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses pre-existing model organisms and LLM-based evaluation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: compute activation differences → apply Patchscope/Logit Lens → compute token relevance → steer model → run interpretability agent → grade agent output. Each step is described in Sections 3 and C.1-C.4."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements state: 'This work was carried out as part of the ML Alignment & Theory Scholars (MATS) program.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: EPFL, Ecole Normale Supérieure Paris-Saclay, Anthropic Fellows Program, Harvard University, MATS."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "MATS is an educational research program focused on AI alignment. While Anthropic has interests in AI safety, the specific findings (narrow finetuning creates detectable biases) do not have a clear commercial stake for any funder."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-5 as the interpretability agent and grader but does not state GPT-5's training data cutoff. The agent could have knowledge about these model organisms from its training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the GPT-5 agent or graders have prior knowledge of the model organisms being evaluated, several of which are from published papers (emergent misalignment, subliminal learning)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The grading agent (GPT-5) could have knowledge about the finetuning objectives of these well-known model organisms from its training data, potentially inflating agent performance. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs reported despite extensive use of GPT-5 for agents and grading, plus running activation extraction on 10000 samples across 33 model organisms."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget stated. The paper trains 20+ SDF models and runs extensive agent evaluations but does not quantify GPU hours or API costs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "While agent evaluations are run 5 times, no seed sensitivity analysis is performed for model finetuning or activation extraction. SDF models are each trained once without reporting seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 'Each configuration is evaluated 5 times with average grades reported' (Section 4.1). Steering uses 5 generations with temperature 1.1 (Section C.3). Token relevance repeated 3 times with shuffled order (Section C.2)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Binary search is described for steering factor α, but no broader hyperparameter search budget is reported for choices like k=5 tokens, middle layer selection, number of pretraining samples (10000), or Patchscope scaling factor ranges."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Steering factor selection is justified via automated coherence grading with binary search (Section C.3). Patchscope scaling factor selected via tournament grading across 30 candidates (Section C.1). Middle layer choice ℓ=⌊L/2⌋ is stated explicitly."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons across 33 organisms, 7 models, and multiple conditions without explicit multiple comparison correction. The Bayesian GLM (Figure 4) partially addresses this through hierarchical modeling but does not cover all comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed and evaluate their own ADL method against blackbox baselines without acknowledging self-evaluation bias. The blackbox baseline uses the same agent architecture, which could disadvantage it."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "ADL agents have access to precomputed activation analyses (requiring substantial compute for extraction across 10000 samples and 33 organisms) while blackbox agents do not. This compute asymmetry is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Central to the paper: it extensively discusses whether model organisms are valid proxies for real-world finetuning (Sections 1, 7) and acknowledges that 'narrow finetuning signals completely overpower any traces from standard chat finetuning,' questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffold comparison issue — the paper evaluates an interpretability method (ADL), not comparing models across different scaffolds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-5 (the agent/grader) was trained on data containing descriptions of these model organisms, which were published before GPT-5's likely training cutoff."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The ADL agent receives activation difference information that inherently provides more signal than the blackbox baseline. While this is the point of the paper, the information asymmetry is not framed as a potential leakage concern."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Multiple organisms from the same family (e.g., 20 SDF organisms, 6 EM organisms) share structural similarities. Non-independence between organisms within a family is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied to verify that the GPT-5 agent does not have prior knowledge of the model organisms being tested."
       }
     }
   },
   "claims": [
     {
       "claim": "ADL-enabled agents perform more than twice as well at identifying broad finetuning objectives and over 30 times better at identifying specific details compared to blackbox baselines.",
       "evidence": "Figure 3 shows ADL agent with i=5 achieves grade≥2 for 91% of organisms vs 39% for blackbox i=50; grade≥4 for 30% vs 1%.",
       "supported": "strong"
     },
     {
       "claim": "Activation differences on the first few tokens of random text encode meaningful information about the finetuning domain.",
       "evidence": "Figure 2a shows 20-80% of top-20 Patchscope tokens are relevant to finetuning domain. Figure 2b shows steering produces text with high cosine similarity to finetuning data. Tested across 33 organisms, 7 models.",
       "supported": "strong"
     },
     {
       "claim": "The detectable bias arises from overfitting to semantically homogeneous finetuning data.",
       "evidence": "Section 5, Figure 7: Causal ablation shows positive CE loss effect on finetuning data (bias is beneficial) and negative/neutral effect on pretraining data (bias hurts generalization) for 2/3 models. Random vector baselines show near-zero effect.",
       "supported": "moderate"
     },
     {
       "claim": "Mixing unrelated data during finetuning can mostly eliminate the detectable bias.",
       "evidence": "Figure 8 shows bias reduction with increasing mixing ratios across three models. At 1:1 ratio, all agents fail to achieve average grade≥2. However, mixing can also reduce FFA scores (Section 6).",
       "supported": "strong"
     },
     {
       "claim": "Narrow finetuning signals dominate even when overlaid on substantial base-to-chat transformation.",
       "evidence": "Figure 5 shows no significant difference in agent performance when comparing base→finetuned vs chat→finetuned models (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "These findings raise concerns about using narrowly finetuned model organisms as realistic proxies for broader finetuning.",
       "evidence": "Section 7 argues that narrow finetuning signals are 'artificially strong compared to realistic post-training scenarios.' Domain finetuning (Section D.5) shows reduced but still partially detectable bias.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Narrow finetuning creates strong, easily detectable biases in LLM activations visible in the first few tokens of unrelated text. An interpretability agent using Activation Difference Lens (Patchscope + steering) identifies finetuning objectives with 91% success rate vs 39% for blackbox baselines across 33 model organisms and 7 architectures. These biases appear to stem from overfitting to semantically homogeneous data and can be mostly mitigated by mixing in unrelated pretraining data, though this may reduce finetuning effectiveness. The findings question the validity of narrowly finetuned model organisms as proxies for studying broader finetuning effects.",
   "red_flags": [
     {
       "flag": "LLM-graded evaluation pipeline",
       "detail": "All evaluation relies on LLM graders (gpt-5-mini, gpt-5-nano) and an LLM agent (gpt-5). The paper acknowledges this introduces noise but does not validate against human judgments. The grader could have prior knowledge of the model organisms being evaluated."
     },
     {
       "flag": "Potential contamination of evaluator",
       "detail": "GPT-5 serves as both the interpretability agent and the grader. It may have been trained on descriptions of the model organisms (emergent misalignment, subliminal learning, etc.), which were published papers. This could inflate agent performance, especially for well-known organisms."
     },
     {
       "flag": "Self-evaluation bias",
       "detail": "The authors designed the ADL method, the agent prompts, the grading rubrics, and the evaluation pipeline. The blackbox baseline uses the same agent but without ADL tools, potentially disadvantaging it through prompt design choices optimized for ADL access."
     }
   ],
   "cited_papers": [
     {
       "title": "Emergent misalignment: Narrow finetuning can produce broadly misaligned LLMs",
       "authors": ["Jan Betley", "Daniel Chee Hian Tan", "Niels Warncke", "Anna Sztyber-Betley", "Xuchan Bao", "Martín Soto", "Nathan Labenz", "Owain Evans"],
       "year": 2025,
       "relevance": "Core model organism source for emergent misalignment, directly relevant to AI safety and alignment research."
     },
     {
       "title": "Alignment faking in large language models",
       "authors": ["Ryan Greenblatt", "Carson Denison", "Benjamin Wright"],
       "year": 2024,
       "arxiv_id": "2412.14093",
       "relevance": "Foundational work on alignment faking using model organisms, relevant to AI safety evaluation methodology."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Key work on deceptive AI behaviors and model organisms for safety research."
     },
     {
       "title": "Building and evaluating alignment auditing agents",
       "authors": ["Trenton Bricken", "Rowan Wang", "Sam Bowman"],
       "year": 2025,
       "relevance": "Automated alignment auditing using LLM agents, directly relevant to AI safety evaluation methodology."
     },
     {
       "title": "Subliminal learning: Language models transmit behavioral traits via hidden signals in data",
       "authors": ["Alex Cloud", "Minh Le", "James Chua"],
       "year": 2025,
       "arxiv_id": "2507.14805",
       "relevance": "Model organism for subliminal preference learning in LLMs, relevant to AI safety and training data effects."
     },
     {
       "title": "Modifying LLM beliefs with synthetic document finetuning",
       "authors": ["Rowan Wang", "Avery Griffin", "Johannes Treutlein"],
       "year": 2025,
       "relevance": "Synthetic document finetuning methodology for implanting false facts, core technique used in this paper."
     },
     {
       "title": "Towards eliciting latent knowledge from LLMs with mechanistic interpretability",
       "authors": ["Bartosz Cywiński", "Emil Ryd", "Senthooran Rajamanoharan", "Neel Nanda"],
       "year": 2025,
       "arxiv_id": "2505.14352",
       "relevance": "Using interpretability to elicit latent knowledge from LLMs, directly related to AI safety evaluation."
     },
     {
       "title": "Overcoming sparsity artifacts in crosscoders to interpret chat-tuning",
       "authors": ["Julian Minder", "Clément Dumas", "Caden Juang", "Bilal Chugtai", "Neel Nanda"],
       "year": 2025,
       "arxiv_id": "2504.02922",
       "relevance": "Model diffing methodology for understanding finetuning effects, core related work."
     },
     {
       "title": "Patchscopes: A unifying framework for inspecting hidden representations of language models",
       "authors": ["Asma Ghandeharioun", "Avi Caciularu", "Adam Pearce", "Lucas Dixon", "Mor Geva"],
       "year": 2024,
       "relevance": "Core interpretability technique (Patchscope) used as the primary analysis tool in this paper."
     },
     {
       "title": "FIND: A function description benchmark for evaluating interpretability methods",
       "authors": ["Sarah Schwettmann", "Tamar Rott Shaham", "Joanna Materzynska"],
       "year": 2023,
       "relevance": "Benchmark for automated interpretability evaluation, methodological foundation for the agent-based evaluation approach."
     },
     {
       "title": "Persona features control emergent misalignment",
       "authors": ["Miles Wang", "Tom Dupré la Tour", "Olivia Watkins"],
       "year": 2025,
       "arxiv_id": "2506.19823",
       "relevance": "Mechanistic analysis of emergent misalignment through persona features, relevant to AI safety interpretability."
     },
     {
       "title": "A survey on large language model based autonomous agents",
       "authors": ["Lei Wang", "Chen Ma", "Xueyang Feng"],
       "year": 2024,
       "doi": "10.1007/s11704-024-40231-1",
       "relevance": "Survey of LLM-based autonomous agents, relevant to understanding agentic AI capabilities and evaluation."
     }
   ]
 }