ai-research-survey

scan.json (22000B)

---

 {
   "paper": {
     "title": "Concept Influence: Leveraging Interpretability to Improve Performance and Efficiency in Training Data Attribution",
     "authors": ["Matthew Kowal", "Gonçalo Paulo", "Louis Jaburi", "Tom Tseng", "Lev E McKinney", "Stefan Heimersheim", "Aaron David Tucker", "Adam Gleave", "Kellin Pelrine"],
     "year": 2026,
     "arxiv_id": "2602.14869"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. The paper references the Persona Vectors repository (Chen et al., 2025) and curvlinops library but does not release its own code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: emergent misalignment datasets from the Persona Vectors repository (Chen et al., 2025), OpenAssistant v1 (OASST1; Köpf et al., 2023), and publicly available SAEs from Neuronpedia."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. The paper mentions curvlinops library and specific models but does not provide reproducible environment specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. Implementation details in Appendix B describe methodology but not how to reproduce the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 shows error bars (shaded regions) for the OASST1 post-training experiment. The text mentions results 'within error bars' (Section 4.2)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Claims like 'Concept Influence outperform classical influence functions across all domains' are based on visual comparison of bar charts without any tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports concrete effect sizes with context: e.g., '20× faster' speedup (Table 1), evil scores going from ~0.5 to ~2.3, MTBench scores from 38% to 67%, and specific timing comparisons (57s vs 1170s)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the choice of sample sizes (e.g., why 1,000 examples for efficiency comparison, why top-5 most influential points, why specific dataset sizes)."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The emergent misalignment experiments (Figures 2, 11-14) show single-run bar charts without error bars or variance across runs. Only Figure 5 shows uncertainty bands. No mention of multiple runs or seeds for the main EM experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against classical influence functions (Grosse et al., 2023), Projection Difference (Chen et al., 2025), and Vector Filter across all experiments."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent methods: Persona Vectors/Projection Difference (Chen et al., 2025), influence functions scaled to LLMs (Grosse et al., 2023), and the IF-GUIDE approach (Coalson et al., 2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically compares variants (Vector Filter, Projection Difference, Concept Influence, Influence Functions) which are theoretically connected as successive approximations. Layer subsampling analysis in Appendix B/Figure 6 also serves as an ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: evil trait score, sycophancy score, MTBench instruction-following score, precision/recall/AUC for misalignment detection, correlation between methods, and wall-clock time."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluations use LLM judges (OpenAI API-based) and automated benchmarks. Given that the paper makes claims about semantic quality of attributions (e.g., Figure 4 qualitative comparison), human evaluation would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The emergent misalignment evaluation uses test examples outside the finetuning domain (the core premise of emergent misalignment). The OASST1 experiment uses MTBench as an independent test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (Misaligned Opinions, Bad Medical Advice, Insecure Code, GSM8k Mistakes), per model (Qwen2.5-7B, Llama3.1-8b), and per trait (evil, sycophancy)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure cases: 'projection-difference scores become less reliable than influence-based methods' under distribution shift (Section 4.1.1), and Vector Filter 'sometimes fails entirely to identify harmful examples' for out-of-domain datasets like GSM8K."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that Concept Influence 'marginally underperforms standard influence functions at very small data fractions' in the OASST1 experiment (Section 4.2), and that vector-based methods fail in out-of-distribution settings."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about comparable performance with improved scalability are supported by Figures 2, 5 and Table 1. The 'order-of-magnitude faster' claim is supported by the 20x speedup in Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about data filtering effects and validates them through retrain-and-evaluate experiments (remove data, retrain, measure outcome change). This is a controlled interventional design adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim broadly about 'Training Data Attribution' but experiments are limited to two 7-8B models (Qwen2.5-7B and Llama3.1-8b) on specific EM datasets and one post-training dataset. The Discussion acknowledges 'scaling these experiments across model sizes' as future work but the title/abstract do not bound claims accordingly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.2 discusses that vector-based and gradient-based methods 'are measuring fundamentally different notions of influence' and explains why each family captures different information. The discussion section explicitly addresses when each method is expected to succeed or fail."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: Qwen2.5-7B, Llama3.1-8b, Qwen2-7B-Base, Gemma-3-9B Instruct. These include size specifications. However, exact snapshot dates for the OpenAI LLM judge API are not specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses an OpenAI API-based LLM judge for evaluation but does not provide the actual judge prompts. It references 'the weighted log-probs trick (Betley et al., 2025)' and 'the implementation from Chen et al. [2025]' without reproducing the prompts used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B reports key hyperparameters: damping λ=10^-3, batch size 1, max sequence length 1536 tokens, 5000 examples for Kronecker factors, LoRA finetuning for single epoch, SAE firing threshold 0.1, SAE layer 20."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a standard ML pipeline paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes dataset construction: 50% benign/50% misaligned splits, English subset of OASST1, and explains how test queries are sampled on-policy from finetuned models and ranked by trait score (Appendix B)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The 'Future work' paragraph in Section 5 mentions extensions but does not discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The future work section mentions scaling to other model sizes and multimodal settings but frames these as opportunities rather than limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its claims to the tested models, datasets, or settings despite testing only 7-8B parameter models on specific synthetic and real-world datasets."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw influence scores, model outputs, and intermediate results are not made available. Only aggregated results in figures are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are clearly described: synthetic EM datasets from Chen et al. (2025) with 50/50 benign/misaligned splits, OASST1 corpus details from Köpf et al. (2023), and dataset examples in Appendix C."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from existing public datasets and model-generated outputs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: finetune model → compute influence scores → rank examples → filter top/bottom K → retrain → evaluate with LLM judge. Appendix B provides implementation details."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. Authors are affiliated with FAR.AI, EleutherAI, and University of Toronto but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: FAR.AI, EleutherAI, University of Toronto."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. FAR.AI is an AI safety research organization which has a stake in the safety-relevant outcomes of this work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates pre-trained models (Qwen2.5-7B, Llama3.1-8b) on benchmarks but does not state training data cutoff dates. Relevant because the EM datasets and OASST1 could overlap with training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation benchmarks or EM datasets could have been in the pre-training data of the models used."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "OASST1 was published in 2023 and both Qwen2.5 and Llama3.1 were trained after that. The paper does not discuss potential contamination of this benchmark in the pre-training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports wall-clock time for each method (57s for Vector Filter, 142s for Projection Difference, 1170s for Concept Influence, 1161s for Influence Functions) on 1000 examples, with speedup factors."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (GPU hours, hardware used, total API spend for LLM judge) is stated. The paper does not specify what GPU was used for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "Concept Influence outperforms classical influence functions across all emergent misalignment domains and traits at filtering misaligned data.",
       "evidence": "Figure 2 shows filtering results for Qwen2.5-7B across four EM datasets. Section 4.1.1 states this holds across additional models (Llama3.1-8b) and traits (sycophancy) in Appendix D.",
       "supported": "moderate"
     },
     {
       "claim": "Probe-based attribution methods (Vector Filter, Projection Difference) are first-order approximations of Concept Influence and achieve over 20x speedup.",
       "evidence": "Section 3.3-3.4 provides theoretical derivation. Table 1 shows 20.4x speedup for Vector Filter and 8.2x for Projection Difference on 1000 examples.",
       "supported": "strong"
     },
     {
       "claim": "Vector Filter with only 5% of the OASST1 dataset achieves the best safety-capability tradeoff, matching full dataset instruction-following performance (MTBench 67) with evil score of 0.8 vs 2.3.",
       "evidence": "Figure 5 and Section 4.2 present these results for Qwen2-7B-Base fine-tuned on OASST1.",
       "supported": "moderate"
     },
     {
       "claim": "Vector-based methods fail under distribution shift while gradient-based methods remain robust.",
       "evidence": "Section 4.1.1: 'for out-of-domain datasets such as GSM8K, simple vector filtering sometimes fails entirely.' Figure 3 shows correlation structure between method families.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces Concept Influence, which attributes model behavior to semantic directions (probes, SAE features) rather than individual test examples, improving training data attribution for safety-relevant behaviors. Probe-based methods are shown to be first-order approximations of Concept Influence, achieving 20x speedup with competitive performance. On synthetic emergent misalignment benchmarks with Qwen2.5-7B and Llama3.1-8b, Concept Influence outperforms classical influence functions at identifying misaligned data. On real-world post-training with OASST1, Vector Filter with 5% of data matches full-dataset instruction-following while substantially reducing harmful outputs.",
   "red_flags": [
     {
       "flag": "No variance/error bars on main experiments",
       "detail": "The emergent misalignment filtering experiments (Figure 2, the core results) show single-run bar charts without error bars or multiple seeds. Only the OASST1 experiment (Figure 5) includes uncertainty bands. Given that LoRA finetuning and LLM judge scoring can be stochastic, the absence of variance reporting on the main results is a concern."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated limitations or threats-to-validity discussion. The future work paragraph frames limitations as opportunities rather than acknowledging constraints of the current evaluation."
     },
     {
       "flag": "LLM judge as sole evaluator",
       "detail": "All evaluations of misalignment traits (evil, sycophancy) rely entirely on an OpenAI API-based LLM judge. The judge prompts are not provided, and no human evaluation validates that the judge scores are meaningful. The reliability of LLM judges for measuring abstract traits like 'evilness' is not established."
     }
   ],
   "cited_papers": [
     {
       "title": "Emergent misalignment: Narrow finetuning can produce broadly misaligned llms",
       "authors": ["Jan Betley", "Daniel Tan", "Niels Warncke", "Anna Sztyber-Betley", "Xuchan Bao", "Martín Soto", "Nathan Labenz", "Owain Evans"],
       "year": 2025,
       "arxiv_id": "2502.17424",
       "relevance": "Core benchmark for evaluating training data attribution methods on misalignment detection in LLMs."
     },
     {
       "title": "Studying large language model generalization with influence functions",
       "authors": ["Roger Grosse", "Juhan Bae", "Cem Anil", "Nelson Elhage"],
       "year": 2023,
       "arxiv_id": "2308.03296",
       "relevance": "Key baseline: scaling influence functions to LLMs, providing the EK-FAC approach this paper builds upon."
     },
     {
       "title": "Sleeper agents: Training deceptive llms that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Related work on persistent deceptive behaviors in LLMs that motivates training data attribution for safety."
     },
     {
       "title": "Persona vectors: Monitoring and controlling character traits in language models",
       "authors": ["Runjin Chen", "Andy Arditi", "Henry Sleight", "Owain Evans", "Jack Lindsey"],
       "year": 2025,
       "arxiv_id": "2507.21509",
       "relevance": "Direct baseline providing Projection Difference method and evaluation infrastructure used in this paper."
     },
     {
       "title": "If-guide: Influence function-guided detoxification of llms",
       "authors": ["Zachary Coalson", "Juhan Bae", "Nicholas Carlini", "Sanghyun Hong"],
       "year": 2025,
       "relevance": "Related work applying influence functions specifically to LLM detoxification."
     },
     {
       "title": "Model organisms for emergent misalignment",
       "authors": ["Edward Turner", "Anna Soligo", "Mia Taylor", "Senthooran Rajamanoharan", "Neel Nanda"],
       "year": 2025,
       "relevance": "Extends emergent misalignment observations across different datasets and models."
     },
     {
       "title": "Convergent linear representations of emergent misalignment",
       "authors": ["Anna Soligo", "Edward Turner", "Senthooran Rajamanoharan", "Neel Nanda"],
       "year": 2025,
       "relevance": "Discovers directions in activation space representing misaligned behavior, directly related to concept vectors used in this work."
     },
     {
       "title": "Persona features control emergent misalignment",
       "authors": ["Miles Wang", "Tom Dupré la Tour", "Olivia Watkins"],
       "year": 2025,
       "relevance": "Demonstrates persona features controlling emergent misalignment, closely related to the concept-based attribution approach."
     },
     {
       "title": "Do influence functions work on large language models?",
       "authors": ["Zhe Li", "Wei Zhao", "Yige Li", "Jun Sun"],
       "year": 2024,
       "arxiv_id": "2409.19998",
       "relevance": "Evaluates reliability of influence functions for LLMs, finding large influence scores can correspond to negligible semantic changes."
     }
   ]
 }