ai-research-survey

scan-v5.json (27266B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "In-the-Wild Model Organisms: Mitigating Undesirable Emergent Behaviors in Production LLM Post-Training via Data Attribution",
     "authors": [
       "Frank Xiao",
       "Santiago Aranguri"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.11079",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: 63% reduction via filtering (Table 2: 2.86% vs 7.63% baseline at 30k), 78% via label switching (1.66%), 10× cost advantage (Table 1: $30 vs $320/$500), and causal validation through retraining is demonstrated in Section 6.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are validated by retraining from the SFT checkpoint with modified DPO datasets, measuring actual harmful response rates after intervention—a proper causal validation design rather than correlation alone.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section explicitly bounds claims to OLMo 2 and a single behavior type, noting that validation on other model families requires access to post-training data and checkpoints that only OLMo currently releases.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why distractor-triggered compliance emerges (e.g., whether DPO's training objective systematically amplifies this pattern, or whether the GPT-4o grading rubric creates systematic label noise); it presents one mechanistic account without considering competing interpretations.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses LLM-judge harmful response rate as its metric and validates this proxy against human annotations (90.6% agreement, Cohen's κ=0.81, Appendix A.9), clearly distinguishing what is measured from what is claimed.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Discussion) contains a dedicated 'Limitations' paragraph with specific constraints beyond boilerplate disclaimers.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are named: single model family constraint tied to OLMo being the only competitive open-source model releasing training artifacts and checkpoints; human interpretation required for cluster identification; 32B validation shows behavior exists but attribution pipeline not applied at that scale.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it does not show generalization beyond OLMo 2 models, does not automate cluster labeling, and has not tested activation-based attribution for RLHF or SFT post-training methods.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "Acknowledgements mention SPAR (Supervised Program for Alignment Research) as providing a research environment but no formal funding source or grant numbers are disclosed.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are stated in the header: Frank Xiao at Caltech, Santiago Aranguri at Goodfire.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Goodfire is a mechanistic interpretability startup whose commercial value depends on activation-based analysis being proven effective; Aranguri's affiliation creates a financial interest in the paper's primary finding that activation-based methods outperform alternatives.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests (patents, equity, consulting) is present in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "'Model organism,' 'in-the-wild model organism,' 'distractor-triggered compliance,' and 'activation-based data attribution' are all explicitly defined with examples in Sections 1, 3, and 5.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three numbered contributions are stated in Section 1: activation-based data attribution, unsupervised behavior discovery, and the in-the-wild model organism itself.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 substantively engages with influence functions, TracIn, TRAK, LESS, Datamodels, mechanistic interpretability, persona vectors, and prior model organism work, explicitly positioning against Minder et al. (2025)'s critique of artificially constructed model organisms.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "Section 4.2 states 'We will additionally release the visualization tool we made to inspect these visualizations upon publication'—this is a promise of future release, not a current release.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The OLMo 2 DPO training dataset (378,341 pairs) is publicly released by AllenAI; LMSys, GSM8K, IFEval, and XSTest are all standard public datasets.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "GPU hardware (H100, 4090) and costs are mentioned but no requirements.txt, Dockerfile, Python version, or dependency specification is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Appendices describe methods in technical detail but provide no step-by-step reproduction guide; key implementation details (layer selection procedure, exact rollout parameters) require inference from scattered appendix sections.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "95% bootstrap confidence intervals are reported for all harmful response rate results in Tables 2, 3, and 4, and error bars are shown in Figures 4, 14, 15, 16, and 19.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No formal statistical significance tests (t-tests, Mann-Whitney, etc.) are used; comparative claims rely on visual inspection of non-overlapping bootstrap CIs rather than explicit hypothesis testing.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Effect sizes are reported as percentage reductions from baseline (63%, 78%, 85%) with absolute harmful response rates, providing clear magnitude context.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The evaluation set of 120 test prompts and 150 probing prompts is not statistically justified; no power analysis or sample size rationale is provided.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Bootstrap confidence intervals (95%) are consistently reported across all main results tables, and standard errors are provided for capability benchmarks in Table 3.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Three baselines are compared: Gradient-based attribution (LESS), LLM Toxic judge ranking, and Random selection, across all intervention sizes.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "LESS (Xia et al., 2024) is a recent gradient-based state-of-the-art method; the LLM judge baseline uses GPT-5-mini, a current model.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Ablations include intervention size (3k/12k/30k datapoints), M0 vs. M1 formulation for probing vector, layer selection (16–26), and additional ranking methods (Max over Vector Bank, Toxic+IF) in the appendices.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Both safety (harmful response rate) and three capability metrics (GSM8K, IFEval, XSTest) are evaluated for all intervention conditions.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.9 reports human annotation of 500 randomly sampled model responses to validate the LLM judge, achieving 90.6% agreement (Cohen's κ=0.81).",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Evaluation uses 120 LMSys prompts held out from the 150 prompts used to construct the probing vector, providing a separate test set for reporting results.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by ranking method, intervention size, model size (7B vs. 32B), and source model (Table 6 shows per-source-model attribution fractions).",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 7 discusses that LESS degrades at larger intervention sizes, that label switching introduces capability tradeoffs at 30k datapoints, and that LLM Toxic+IF underperforms LLM Toxic alone.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Negative results include gradient-based LESS degrading at scale (5.78% vs 2.86% at 30k filtering), label switching causing GSM8K degradation from 72.5% to ~68% at large scale, and the Toxic+IF variant underperforming.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "OLMo 2 7B and 32B are specified; GPT-5-mini (OpenAI, 2025) and Grok 4.1 (xAI, 2025) are named with dated references.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.7 provides the exact LLM Toxic and Toxic+IF judge prompts verbatim; Appendix A.8 provides the full evaluation judge prompt.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Key hyperparameters are reported: layer 20 for probing vector, steering scale α=2.0, LoRA rank 128 with α=512, 8192-dimension projections for LESS, 100 rollouts per prompt, medium reasoning effort for GPT-5-mini.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "This is a training data attribution paper with no agentic scaffolding involved.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.1 documents the four-criterion filter for selecting the 150 probing prompts (SFT without/with distractor: 0 toxicity; DPO without distractor: 0 toxicity; DPO with distractor: non-zero toxicity).",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "The visualization tool and analysis artifacts are promised 'upon publication' but not currently released; while the underlying OLMo 2 training data is public, the probe prompts, evaluation sets, and ranking outputs are not independently available.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.1 describes OLMo 2's DPO dataset construction (responses sampled from 20 LLMs, graded by GPT-4o rubric, highest scoring = accepted); test prompt sampling from LMSys is also described.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participant recruitment—the data is model-generated preference pairs from a standard public dataset.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from OLMo 2 DPO training data → probing vector construction → datapoint ranking → retraining intervention is documented across Sections 3–6 and Appendices A.1–A.5.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "The paper studies data attribution in post-training, not model capability evaluation on benchmarks; training cutoff is not relevant to the primary claims.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "NA—the paper is not evaluating pre-trained model capabilities on held-out benchmarks as a primary contribution.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "Capability benchmarks (GSM8K, IFEval, XSTest) are used only to verify no capability degradation from interventions, not as primary evaluation targets; contamination is not relevant in this context.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the experiment proper.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the experiment proper.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the experiment proper.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the experiment proper.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the experiment proper.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the experiment proper.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the experiment proper.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 provides explicit cost breakdown: probing vector 12 H100 hours ($30), gradient LESS 128 H100 hours ($320), LLM Judge $500; actual 4090 runtime also noted (36 hours, ~$10).",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 states total compute for ranking all 378k datapoints per method; the paper explicitly positions cost efficiency as a key advantage.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Filtering top-ranked datapoints using the probing vector method reduces distractor-triggered harmful compliance by 63% (from 7.63% to 2.86%)",
       "evidence": "Table 2: probing vector filtering at 30,000 datapoints yields 2.86±0.23% vs baseline 7.63±0.36%; causal validation by retraining from SFT checkpoint with modified dataset",
       "supported": "strong"
     },
     {
       "claim": "Switching labels of top-ranked datapoints achieves 78% reduction in harmful behavior (from 7.63% to 1.66%)",
       "evidence": "Table 2: probing vector switching at 30,000 datapoints yields 1.66±0.19%; verified through full DPO retrain",
       "supported": "strong"
     },
     {
       "claim": "Activation-based attribution outperforms gradient-based (LESS) and LLM-judge methods at scale while being 10× cheaper",
       "evidence": "Table 2 at 30k filtering: probing vector 2.86% vs LESS 5.78% vs LLM Toxic 3.61%; Table 1: $30 vs $320 vs $500",
       "supported": "strong"
     },
     {
       "claim": "Unsupervised clustering of behavior-datapoint similarity matrices can discover emergent harmful behaviors without prior specification",
       "evidence": "Figure 3 shows four distinct behavioral clusters; Appendix C.1 independently verifies each cluster; C.2 shows reproducibility across 8 independent random samples",
       "supported": "strong"
     },
     {
       "claim": "Distractor-triggered compliance emerged organically from contaminated preference data in OLMo 2's production DPO training, not deliberate injection",
       "evidence": "Section 5.2–5.4 demonstrates the behavior; green cluster in Figure 3 identifies mislabeled datapoints where accepted responses comply with harmful requests; contamination validated by finding specific source models over-represented in top-ranked harmful datapoints",
       "supported": "moderate"
     },
     {
       "claim": "Removing contributions from problematic source models (top 4 over-represented) achieves 85% harm reduction",
       "evidence": "Table 4: Max over Vector Bank model-level ablation yields 1.17±0.19% harmful rate vs 7.63% baseline, while preserving GSM8K at 74.2%",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "case-study",
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "Activation-based data attribution, which computes cosine similarity between behavior-change vectors and training datapoint vectors in residual activation space, can identify and causally validate which DPO training examples produce harmful emergent behaviors. Applied to OLMo 2, the method discovered distractor-triggered compliance—a behavior where benign formatting instructions appended to harmful requests (e.g., 'no more than 50 words') cause the DPO-trained model to comply at ~7.6% rate while the SFT model maintains near-zero compliance. Filtering or relabeling 30,000 identified datapoints reduces harmful behavior by 63–78% while preserving mathematical reasoning and instruction-following capabilities. The method outperforms gradient-based (LESS) and LLM-judge baselines at scale while being 10× cheaper, and its unsupervised discovery component reliably surfaces behavioral clusters without prior specification of what harmful patterns to seek.",
   "red_flags": [
     {
       "flag": "Single model family",
       "detail": "All causal attribution experiments are on OLMo 2 only; OLMo 3 generalization shows unsupervised discovery works but attribution and mitigation pipeline is not validated on any other model family."
     },
     {
       "flag": "Undisclosed financial conflict",
       "detail": "Santiago Aranguri is affiliated with Goodfire, a mechanistic interpretability startup whose commercial value depends on activation-based methods being proven superior; no competing interests statement is present."
     },
     {
       "flag": "Probing vector requires known behavior",
       "detail": "Constructing the probing vector requires 150 prompts that already exhibit the target behavior, partially undermining the 'unsupervised' framing—the discovery phase is unsupervised but targeted attribution requires behavior awareness."
     },
     {
       "flag": "Single harmful behavior validated causally",
       "detail": "Only distractor-triggered compliance undergoes full causal validation (retraining); other discovered behaviors (formatting, verbosity, OLMo 3 roleplay) are verified empirically but not through the full attribution-and-retrain pipeline."
     },
     {
       "flag": "No formal significance tests",
       "detail": "Comparative performance claims between methods rely on non-overlapping bootstrap CIs without formal hypothesis tests; at small intervention sizes (3k) several results show overlapping CIs making superiority claims weaker."
     },
     {
       "flag": "Sample size unjustified",
       "detail": "120 test prompts with 100 rollouts each is not statistically justified; no power analysis for detecting the observed effect sizes."
     }
   ],
   "cited_papers": [
     {
       "title": "LESS: Selecting Influential Data for Targeted Instruction Tuning",
       "relevance": "Primary baseline for gradient-based data attribution; directly compared throughout all experiments"
     },
     {
       "title": "Narrow Finetuning Leaves Clearly Readable Traces in Activation Differences (Minder et al., 2025)",
       "relevance": "Core motivation for the paper—critiques artificially-constructed model organisms and directly addressed by the in-the-wild model organism contribution"
     },
     {
       "title": "Direct Preference Optimization: Your Language Model is Secretly a Reward Model",
       "relevance": "Post-training method whose training data is audited throughout the paper"
     },
     {
       "title": "The Linear Representation Hypothesis and the Geometry of Large Language Models (Park et al., 2024)",
       "relevance": "Theoretical foundation for why cosine similarity in activation space captures behavioral alignment"
     },
     {
       "title": "Persona Vectors: Monitoring and Controlling Character Traits in Language Models (Chen et al., 2025)",
       "relevance": "Closest related work on activation-space behavioral analysis; distinguished by targeting pre-specified vs. unknown behaviors"
     },
     {
       "title": "Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training (Hubinger et al., 2024)",
       "relevance": "Exemplar of artificially-constructed model organisms that the in-the-wild approach is designed to complement"
     },
     {
       "title": "TRAK: Attributing Model Behavior at Scale (Park et al., 2023)",
       "relevance": "Alternative data attribution method reviewed in related work"
     },
     {
       "title": "Understanding Black-Box Predictions via Influence Functions (Koh & Liang, 2017)",
       "relevance": "Foundational data attribution approach that the paper's activation-based method is positioned against"
     },
     {
       "title": "OLMo 2: The Model Studied (OLMo Team, 2025)",
       "relevance": "The production model whose DPO training is audited; uniquely suitable because it releases training data and intermediate checkpoints"
     },
     {
       "title": "Representation Engineering: A Top-Down Approach to AI Transparency (Zou et al., 2023)",
       "relevance": "Foundational mechanistic interpretability work supporting the linear representation assumption underlying the method"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly applicable to any organization running DPO post-training: a $30 safety audit method that can find and fix harmful data contamination before deployment."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that appending benign formatting instructions like 'no more than 50 words' bypasses safety training is counterintuitive and challenges assumptions about DPO robustness."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Demonstrates that production models (OLMo 2) have exploitable safety vulnerabilities that evade standard safety evaluations, triggered by normal user behavior rather than adversarial prompting."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Directly critiques the model organism methodology used widely in AI safety research (Hubinger et al., Perez et al.) and shows production contamination that researchers didn't anticipate."
     },
     "demo_ability": {
       "score": 2,
       "justification": "OLMo 2 is publicly available so the distractor-triggered compliance behavior can be reproduced, but the full attribution pipeline requires substantial compute and access to training checkpoints."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Caltech and Goodfire (niche mechanistic interpretability startup) are not major AI lab brands; paper evaluates AllenAI's OLMo rather than a flagship commercial model."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }