ai-research-survey

scan.json (27611B)

---

 {
   "paper": {
     "title": "Simple LLM Baselines are Competitive for Model Diffing",
     "authors": ["Elias Kempf", "Simon Schrodi", "Bartosz Cywiński", "Thomas Brox", "Neel Nanda", "Arthur Conmy"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.10371"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "An improved LLM-based baseline for model diffing performs comparably to SAE-based methods on generalization (accuracy, frequency) and interestingness, while consistently producing more abstract hypotheses. Neither method detected a hidden gender assumption finetuned into a model organism, revealing a fundamental limitation of API-only model diffing: behaviors not manifested in outputs cannot be surfaced. The authors propose evaluation desiderata (generalization, interestingness, abstraction level) and operationalize them into metrics, providing the first systematic comparison framework for model diffing methods.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Code will be available on GitHub' (Section 1, footnote 1) but this is a future promise, not a current release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available WildChat dataset (Zhao et al., 2024) and publicly available model organisms (Qwen-2.5-7B-Instruct, gemma-2-9b-it finetuned versions). The finetuned models are from prior published work with public artifacts."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The pipeline is described at a high level but lacks specific commands or scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported in Figure 2 and described in Appendix B.3: 'All confidence intervals reported in Figure 2 are 95% intervals computed over hypotheses using a t-distribution with n−1 degrees of freedom.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares LLM-based vs SAE-based methods and claims comparable performance but does not use statistical significance tests. Comparisons are based on visual inspection of overlapping confidence intervals."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with context throughout, e.g., 'average response length (68.1 vs. 495.8 tokens)', 'Markdown table usage increased from 0.7% to 13.9%', accuracy/frequency percentages with baselines in all tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 1,000 prompts for generation and 500 for testing but does not justify why these sizes were chosen or whether they provide adequate statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.2 reports full distributions with standard deviations (e.g., 'LLM: n=68, μ=90.8%, σ=11.7%') and kernel density estimates across all metrics and experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares LLM-based method against SAE-based method (Jiang et al., 2025), and also includes a KL divergence-based approach in Appendix E."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Both baselines are from 2025 — Dunlap et al. (2025) VibeCheck and Jiang et al. (2025) SAE-based method — which are the most recent API-only model diffing approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The paper modifies both pipelines (e.g., adding prompt context to SAE-based method) but does not ablate these modifications to measure their individual contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five evaluation metrics are used: accuracy, frequency, interestingness, abstraction level, and acceptance rate (Section 3, Figure 1b)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated using LLM judges and autoraters. The paper acknowledges this limitation: 'Manual verification by human experts would be ideal but does not scale' (Section 3.1). Some manual verification is done post-hoc (e.g., regex-verifying table usage) but no systematic human evaluation of hypothesis quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 states 'Evaluation uses a held-out set of 500 prompts' separate from the 1,000 prompts used for generation. Frequency and accuracy are computed on this held-out data (Section 3.2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per experiment (Qwen misalignment, Gemma gender, Gemini revisions) and per metric (accuracy, frequency, interestingness, abstraction, acceptance rate) in Figure 2 and Tables 2-4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Gemma gender assumption experiment (Section 4.1) is an explicit failure case — neither method detected the ground truth behavior. This is analyzed in depth in Appendix D with explanation of why detection failed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that neither method detected the hidden gender assumption (Section 4.1), that KL divergence-based approach only surfaced low-level formatting differences (Appendix E), and that the Gemini experiment showed notably lower accuracy for both methods."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the LLM-based baseline 'performs comparably to the SAE-based method while typically surfacing more abstract behavioral differences,' which is supported by Figure 2 showing overlapping CIs for accuracy/frequency and higher abstraction scores for LLM."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims about method design leading to different abstraction levels (e.g., 'token-level methods may inherently struggle to produce abstract hypotheses' in Section 5) without controlled experiments isolating these factors."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to bound its claims: 'our use of WildChat surfaces general behavioral patterns but may miss domain-specific differences' (Section 5), 'the methods studied here have not been sufficiently stress-tested to serve as standalone audit techniques' (Section 5)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 (Gemma) explicitly considers alternative explanations: whether the method failed or the behavior simply doesn't manifest on WildChat. Appendix D provides extensive analysis confirming the latter explanation. Section 5 discusses prompt distribution dependence."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses the gap between LLM judge verdicts and actual hypothesis quality: 'Automatic evaluation has inherent limitations: LLM judges can be unreliable, and while our metrics provide valuable signal, they cannot guarantee hypothesis quality' (Section 5). The paper also validates proxy metrics against ground truth (e.g., frequency metric aligning with regex-measured difference in Section 4.2)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: 'Qwen-2.5-7B-Instruct', 'gemma-2-9b-it', 'gemini-2.5-flash-lite' and 'gemini-2.5-flash-lite-preview-09-2025' (footnote 2), 'Gemini 2.5 Flash' for judge, 'llama-embed-nemotron-8b' for embeddings, 'LLaMA 3.3 70B' as SAE reader."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full judge system and user prompts are provided in Listings 1 and 2. Full autorater prompts in Listings 3 and 4. The paper states 'Full prompts are available in the released code' for diffing pipelines (Appendix A.2)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Key hyperparameters are missing: temperature/sampling settings for model completions, the 65% threshold for direction assignment is stated but PCA components (128), UMAP components (30), and max tokens (1,024) are the only numerical settings mentioned. No temperature or top-p values for any of the LLM calls."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Both pipelines are described in detail: the LLM-based pipeline's 4-step process (difference extraction → embedding/clustering → summarization → direction assignment) in Appendix A.2, and the SAE-based pipeline's feature extraction and selection process in Appendix A.3."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is documented: prompts sampled from WildChat, model identifier normalization before embedding (Appendix A.2), PCA + UMAP for dimensionality reduction, HDBSCAN for clustering, max-pooling of SAE activations over completion tokens only."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Discussion) contains a dedicated 'Limitations' subsection with substantive discussion of automatic evaluation limitations, prompt distribution dependence, and API-only constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: LLM judge unreliability (with evidence from Appendix D showing judge inconsistency on gender assumption examples), prompt distribution affecting detectable behaviors, inability to assess undetected differences, and the gender assumption experiment as concrete evidence of method limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries: 'the methods studied here have not been sufficiently stress-tested to serve as standalone audit techniques' (Section 5), 'our framework only evaluates discovered hypotheses; it cannot assess if true differences go undetected' (Section 5), focus limited to API-only methods."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data (model completions, judge verdicts, hypothesis sets) is not released. Only aggregated results are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: 1,000 prompts from WildChat, responses generated with max 1,024 new tokens, 500 held-out prompts for evaluation (Section 2). Appendix D describes the extended analysis with 19,855 WildChat prompts."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is WildChat, a standard public dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from prompt collection through hypothesis generation to evaluation is documented in Section 2, Appendix A, and Section 3. Each step (data collection → diffing → judge verification → metric computation) is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding disclosed in Acknowledgments: MATS Program, German Federal Ministry for Research (BMFTR) grant 01GQ2510, German Research Foundation (DFG) grants 417962828 and 539134284."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations listed: University of Freiburg, MATS, IDEAS Research Institute. No authors appear affiliated with companies whose products are being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are academic (German government research grants, DFG) and MATS (AI safety research program). None have financial stake in whether LLM-based or SAE-based methods perform better."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate pre-trained model capability on a benchmark. It uses models to generate completions and then evaluates model diffing methods — the subject is the diffing methodology, not model performance."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale: the paper evaluates diffing methods, not model knowledge on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale: contamination of model training data is not relevant to the claims being made about diffing method quality."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or wall-clock times are reported despite the pipeline involving multiple LLM calls (diffing, judging, autorating with 3 frontier models)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The pipeline uses multiple expensive frontier models (GPT-5.2, Claude Opus 4.5, Grok-4, Gemini 2.5 Flash) but costs are not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. The clustering step (HDBSCAN) and LLM generation are stochastic but results are not reported across multiple random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It appears each experiment was run once. The autorater uses 3 LLMs averaged but this is for variance reduction in scoring, not multiple experimental runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Pipeline hyperparameters (PCA 128 components, UMAP 30 components, 65% direction threshold, 40 SAE hypotheses) appear tuned but no search budget or justification is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Configuration choices (e.g., number of clusters, dimensionality reduction parameters, hypothesis count) are stated but not justified through systematic comparison."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No significance tests are performed, so multiple comparison correction is not applied either. However, multiple metrics are compared across multiple experiments without correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and modify the LLM-based method while comparing it against an external SAE-based method. This potential bias is not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The two methods likely differ substantially in compute requirements (SAE requires running a 70B reader model vs. LLM-based using Gemini Flash) but performance is not compared at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses construct validity of its evaluation framework: Section 3.1 critiques the prior 'judge-verified frequency' metric for conflating frequency and accuracy, and Section B.4 discusses trade-offs between metrics and what they actually measure."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The LLM-based method uses Gemini 2.5 Flash for multiple steps while the SAE-based method uses LLaMA 3.3 70B as reader plus Gemini for summarization. These different scaffolding choices could confound the comparison but this is not discussed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "WildChat data could overlap with training data of the models used for evaluation (Gemini, GPT-5.2, etc.) but this is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLM judge or autoraters could be biased by exposure to model diffing hypotheses in training data."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 1,000 training and 500 test prompts from WildChat share structural similarities or come from similar conversations."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "The improved LLM-based baseline performs comparably to the SAE-based method on generalization (accuracy and frequency) and interestingness.",
       "evidence": "Figure 2 shows overlapping 95% confidence intervals for accuracy and frequency across all three experiments. Interestingness scores are similar (LLM 1.81-2.24 vs SAE 1.83-2.71, Appendix C.2).",
       "supported": "moderate"
     },
     {
       "claim": "The LLM-based method consistently produces more abstract hypotheses than the SAE-based method.",
       "evidence": "Figure 2 and Appendix C.2 show LLM abstraction means of 3.83-4.25 vs SAE means of 2.36-3.46 across all three experiments, with non-overlapping distributions in some cases (Figure 7).",
       "supported": "strong"
     },
     {
       "claim": "API-only model diffing cannot surface behavioral differences that do not manifest in model outputs.",
       "evidence": "Section 4.1 and Appendix D demonstrate that neither method detected the hidden gender assumption, with extended analysis on 513 prompts (Figure 9) confirming the behavior rarely manifests on WildChat prompts.",
       "supported": "strong"
     },
     {
       "claim": "The prior 'judge-verified frequency difference' metric conflates frequency and accuracy, hiding distinct aspects of hypothesis quality.",
       "evidence": "Section 3.1 and Appendix B.1 provide the mathematical decomposition (Equation 4) showing vfd = f(h) · (2·acc(h)−1) with a concrete example of two hypotheses with identical vfd but very different properties.",
       "supported": "strong"
     },
     {
       "claim": "The LLM-based method achieves higher acceptance rates, indicating more robust hypothesis generation.",
       "evidence": "Figure 2 shows higher acceptance rates for LLM across experiments, but confidence intervals sometimes overlap. Section 4.3 states this finding.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No ablation of pipeline modifications",
       "detail": "The authors modify both pipelines from prior work (e.g., adding prompt context to SAE, changing clustering approach for LLM) but never ablate these changes to measure their individual contribution. It's unclear whether improvements come from the modifications or are inherent to the approach."
     },
     {
       "flag": "LLM autorater circularity",
       "detail": "Interestingness and abstraction are evaluated by LLM autoraters (GPT-5.2, Claude Opus 4.5, Grok-4), but the LLM-based diffing method is itself an LLM pipeline. LLMs may systematically rate LLM-generated hypotheses higher on abstraction than SAE-generated ones due to shared linguistic biases."
     },
     {
       "flag": "Single-run results with stochastic pipeline",
       "detail": "The diffing pipeline involves stochastic steps (LLM generation, clustering) but results appear to be from single runs. Hypothesis sets could vary substantially across runs, affecting all downstream metrics."
     },
     {
       "flag": "No cost comparison between methods",
       "detail": "The SAE-based method requires running a 70B reader model while the LLM-based method uses Gemini Flash. Practical cost differences are not reported despite being relevant to the 'practical implications' discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "VibeCheck: Discover and quantify qualitative differences in large language models",
       "authors": ["Lisa Dunlap", "Krishna Mandal", "Trevor Darrell", "Jacob Steinhardt", "Joseph E. Gonzalez"],
       "year": 2025,
       "relevance": "Primary baseline for LLM-based model diffing; proposes clustering-based hypothesis generation from model response differences."
     },
     {
       "title": "Interpretable Embeddings with Sparse Autoencoders: A Data Analysis Toolkit",
       "authors": ["Nick Jiang", "Xiaoqing Sun", "Lisa Dunlap", "Lewis Smith", "Neel Nanda"],
       "year": 2025,
       "arxiv_id": "2512.10092",
       "relevance": "Primary baseline for SAE-based model diffing; uses SAE features with activation frequency differences to identify behavioral differences."
     },
     {
       "title": "Emergent Misalignment: Narrow finetuning can produce broadly misaligned LLMs",
       "authors": ["Jan Betley", "Daniel Tan", "Niels Warncke"],
       "year": 2025,
       "relevance": "Creates model organisms of misalignment used as ground-truth test cases for evaluating model diffing methods."
     },
     {
       "title": "Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates deceptive AI behaviors that persist through safety training, motivating the need for model diffing methods."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "Establishes the LLM-as-a-judge evaluation paradigm used throughout this paper for hypothesis verification."
     },
     {
       "title": "WildChat: 1M ChatGPT Interaction Logs in the Wild",
       "authors": ["Wenting Zhao", "Xiang Ren", "Jack Hessel", "Claire Cardie", "Yejin Choi", "Yuntian Deng"],
       "year": 2024,
       "relevance": "Provides the diverse prompt dataset used for all model diffing experiments in this paper."
     },
     {
       "title": "Towards Monosemanticity: Decomposing Language Models With Dictionary Learning",
       "authors": ["Trenton Bricken", "Adly Templeton", "Joshua Batson"],
       "year": 2023,
       "relevance": "Foundational work on sparse autoencoders for interpretability, underlying the SAE-based model diffing approach."
     },
     {
       "title": "Sparse Crosscoders for Cross-Layer Features and Model Diffing",
       "authors": ["Jack Lindsey", "Adly Templeton", "Jonathan Marcus", "Thomas Conerly", "Joshua Batson", "Christopher Olah"],
       "year": 2024,
       "relevance": "Introduces model diffing concept using sparse crosscoders; foundational motivation for this paper's comparison of diffing approaches."
     },
     {
       "title": "Eliciting secret knowledge from language models",
       "authors": ["Bartosz Cywiński", "Emil Ryd", "Rowan Wang"],
       "year": 2025,
       "arxiv_id": "2510.01070",
       "relevance": "Creates the Gemma gender assumption model organism used as a test case in this paper."
     },
     {
       "title": "Model Organisms for Emergent Misalignment",
       "authors": ["Edward Turner", "Anna Soligo", "Mia Taylor"],
       "year": 2025,
       "arxiv_id": "2506.11613",
       "relevance": "Creates model organisms of emergent misalignment used as ground-truth test cases for model diffing evaluation."
     }
   ]
 }