ai-research-survey

scan.json (20833B)

---

 {
   "paper": {
     "title": "Position: Explaining Behavioral Shifts in Large Language Models Requires a Comparative Approach",
     "authors": ["Martino Ciaperoni", "Marzio Di Vece", "Luca Pappalardo", "Fosca Giannotti", "Francesco Giannini"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.02304"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["theoretical", "case-study"],
   "key_findings": "The paper argues that behavioral shifts in LLMs (from scaling, fine-tuning, RLHF, etc.) require comparative explainability methods (∆-XAI) rather than single-checkpoint XAI. It proposes a framework with 10 desiderata organized into comparability, validity, actionability, and monitoring. A small illustrative experiment on Qwen2.5-0.5B demonstrates that fine-tuning-induced medical misalignment can be localized to late transformer layers via CKA, activation patching, and activation steering.",
   "claims": [
     {
       "claim": "Classical XAI methods are structurally ill-suited to explain behavioral shifts across model checkpoints because they operate on single models in isolation.",
       "evidence": "Argued throughout Section 1 and Section 6 (Alternative Views). The rebuttal notes XAI answers 'why this prediction' not 'what changed after intervention.'",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuning Qwen2.5-0.5B on a narrow dataset induces misaligned medical advice, and this shift is localized to the third-to-last transformer layer.",
       "evidence": "Section 5.2: CKA similarity shows divergence in final 3 layers (Figure 6a), activation patching at third-to-last layer increases cosine similarity to pre-fine-tuning outputs (Figure 6b), and activation steering at that layer shifts unsafe outputs toward safer advice (Figure 6c).",
       "supported": "moderate"
     },
     {
       "claim": "Activation steering using a probe-derived direction at the third-to-last layer can shift misaligned medical advice toward safer outputs without modifying model parameters.",
       "evidence": "Section 5.2 and Figure 6c show qualitative example with α=15 steering. Linear probe achieves >0.95 test accuracy distinguishing pre/post representations.",
       "supported": "weak"
     }
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides an anonymous repository URL: https://anonymous.4open.science/r/ComparativeXAI4LLMs-7CCB for reproducing the experiment (Section 5.2)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 15 prompts for the experiment and 400 expanded prompts generated by GPT-5.2, but does not provide these datasets separately. They may be in the anonymous repo but this is not explicitly stated."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency versions are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper links to a repository but does not include step-by-step reproduction instructions in the paper itself. The experimental description in Section 5.2 is a narrative, not a reproducible protocol."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Figure 6b shows box plots but no confidence intervals or error bars on the main results. The CKA and cosine similarity results are reported as point estimates or aggregates without uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. The activation patching results (Figure 6b) compare distributions visually via box plots but no formal test is applied."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes are reported. The paper shows qualitative and visual comparisons (CKA curves, box plots, steering examples) without quantifying effect magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The experiment uses only 15 prompts with no justification for this sample size. The 400 expanded prompts for probe training are also unjustified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance across runs is reported. Results appear to be from single runs. The box plots in Figure 6b show spread across prompts but not across experimental repetitions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "This is a position paper proposing a framework. The illustrative experiment demonstrates the framework but does not claim superiority over alternatives, so baselines are not structurally required."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No baselines needed for a position paper with illustrative experiments."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper proposes a framework, not a system with separable components. The illustrative experiment demonstrates different analysis techniques but is not presenting a system to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The experiment uses multiple metrics: CKA similarity (Figure 6a), cosine similarity of sentence embeddings (Figure 6b), and probe classification accuracy (>0.95). Multiple analysis perspectives (feature attribution, representation similarity, activation patching, activation steering) are applied."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the steering or patching outputs is conducted. The medical advice quality is assessed only via automated metrics (cosine similarity) and qualitative cherry-picked examples."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "The illustrative experiment is not a benchmark evaluation requiring held-out test sets. It demonstrates analysis techniques on a small prompt set."
       },
       "per_category_breakdown": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a position paper with a small illustrative experiment."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases of the ∆-XAI approach are shown. The paper only presents successful demonstrations of localization and steering."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. Every analysis stage (attribution, CKA, patching, steering) shows the expected positive outcome."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the paper formulates a ∆-XAI framework with desiderata, introduces possible pipelines, and provides a concrete experiment. All three are present in the paper (Sections 3-5)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims about activation patching and steering ('reveals the internal representational changes responsible for this misalignment'), but the experiment uses only 15 prompts with no controls for prompt selection bias, no statistical tests, and no bidirectional validation (mentioned as possible but not performed)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The framework is presented as general for all LLM behavioral shifts, but the sole experiment uses a 0.5B parameter model (Qwen2.5-0.5B) with 15 prompts on one specific behavior (medical advice). The paper does not explicitly bound its claims to this narrow setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Alternative Views) explicitly addresses two counter-positions: that single-checkpoint XAI methods suffice, and that evaluation without explanation is enough. Each receives a substantive rebuttal."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses cosine similarity of sentence embeddings as a proxy for 'safer behavior' and CKA for 'representational change' without discussing the gap between these proxies and the actual constructs of safety and mechanistic understanding."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'instruction-tuned Qwen2.5 model with 0.5 billion parameters' and 'all-MiniLM-L6-v2 sentence-transformer' (Section 5.2). The Qwen version is named with size."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 15 prompts and 400 expanded prompts but does not provide the actual prompt texts in the paper. Only one example prompt appears in Figure 6c. The full set may be in the anonymous repo but is not in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Steering parameter α=15 is stated, but fine-tuning hyperparameters, decoding parameters (temperature, top-p), and Integrated Gradients settings are not reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The experiment applies standard ML analysis techniques."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the 15 prompts were selected or how the 400 GPT-5.2-expanded prompts were generated (what prompt was used to generate them, what filtering was applied)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The closest is Section 6 (Alternative Views) which addresses counter-arguments but does not discuss limitations of the framework or experiment."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The tiny sample size (15 prompts), single model, and single behavior type are not acknowledged as limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The framework is presented broadly without bounding its applicability to specific model sizes, architectures, or shift types."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental data (prompts, model outputs, attribution scores, CKA values) is not provided in the paper. An anonymous repository is linked but its contents are not described."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The origin of the 15 medical prompts is not described. The paper says '15 prompts that request medical advice' without explaining how they were selected or constructed."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses model-generated outputs only."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from prompt selection to final results is not documented with sufficient detail. The expansion from 15 to 400 prompts via GPT-5.2 is mentioned but the generation process is not described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is provided in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Scuola Normale Superiore, ISTI-CNR, and University of Pisa."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. The experiment fine-tunes a model and analyzes its internal representations; it is not testing benchmark performance."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The experiment analyzes behavioral shifts, not benchmark scores."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Position/theoretical paper. The illustrative experiment is not proposing a method whose cost matters for deployment."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Position/theoretical paper with a small illustrative experiment. Compute budget is not a meaningful concern."
       }
     }
   },
   "red_flags": [
     {
       "flag": "Tiny sample size for empirical claims",
       "detail": "The concrete experiment uses only 15 prompts to demonstrate CKA, activation patching, and attribution analysis. This is far too small to draw reliable conclusions about where behavioral shifts are localized, yet the paper presents the results as evidence for the framework's utility."
     },
     {
       "flag": "Cherry-picked qualitative examples",
       "detail": "The activation steering result (Figure 6c) shows a single cherry-picked prompt-response pair. No systematic evaluation of steering quality across prompts is provided."
     },
     {
       "flag": "Missing bidirectional validation",
       "detail": "The paper mentions that bidirectional patching (Mpost→Mpre) 'could' be done as further validation but does not actually perform it, despite presenting the pipeline as providing causal evidence (D6)."
     },
     {
       "flag": "No limitations section",
       "detail": "For a position paper proposing a new framework, there is no discussion of the framework's own limitations, the conditions under which it might fail, or the scalability challenges of applying ∆-XAI to large models."
     }
   ],
   "cited_papers": [
     {
       "title": "Emergent misalignment: Narrow finetuning can produce broadly misaligned LLMs",
       "authors": ["J. Betley", "D. C. H. Tan", "N. Warncke", "A. Sztyber-Betley", "X. Bao", "M. Soto", "N. Labenz", "O. Evans"],
       "year": 2025,
       "relevance": "Demonstrates that narrow fine-tuning can induce broad safety failures across unrelated domains, a key motivation for ∆-XAI."
     },
     {
       "title": "Model organisms for emergent misalignment",
       "authors": ["E. Turner", "A. Soligo", "M. Taylor", "S. Rajamanoharan", "N. Nanda"],
       "year": 2025,
       "arxiv_id": "2506.11613",
       "relevance": "Provides the controlled setting used in the paper's experiment; shows emergent misalignment across model sizes and architectures."
     },
     {
       "title": "Natural emergent misalignment from reward hacking in production rl",
       "authors": ["M. MacDiarmid", "B. Wright", "J. Uesato", "E. Hubinger"],
       "year": 2025,
       "arxiv_id": "2511.18397",
       "relevance": "Demonstrates that Claude can engage in reward hacking that generalizes into lying and sabotage, motivating mechanistic explanation of behavioral shifts."
     },
     {
       "title": "Emergent abilities of large language models",
       "authors": ["J. Wei", "Y. Tay", "R. Bommasani", "C. Raffel"],
       "year": 2022,
       "relevance": "Seminal paper on emergent abilities in LLMs, a core phenomenon that ∆-XAI aims to explain."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["R. Schaeffer", "B. Miranda", "S. Koyejo"],
       "year": 2023,
       "relevance": "Counter-argument that emergent abilities may be artifacts of evaluation metrics rather than intrinsic model properties."
     },
     {
       "title": "Convergent linear representations of emergent misalignment",
       "authors": ["A. Soligo", "E. Turner", "S. Rajamanoharan", "N. Nanda"],
       "year": 2025,
       "arxiv_id": "2506.11618",
       "relevance": "Extracts directions from activation differences associated with safety-relevant shifts, directly related to ∆-XAI mechanistic interventions."
     },
     {
       "title": "HarmBench: A standardized evaluation framework for automated red teaming and robust refusal",
       "authors": ["M. Mazeika", "L. Phan", "X. Yin", "A. Zou"],
       "year": 2024,
       "relevance": "Standardized safety evaluation framework for LLMs, representing the behavioral-testing approach that ∆-XAI aims to complement."
     },
     {
       "title": "Towards automated circuit discovery for mechanistic interpretability",
       "authors": ["A. Conmy", "A. Mavor-Parker", "A. Lynch", "S. Heimersheim", "A. Garriga-Alonso"],
       "year": 2023,
       "relevance": "Activation patching for mechanistic interpretability, a key technique family in the ∆-XAI framework."
     },
     {
       "title": "Cot is not explainability",
       "authors": ["F. Barez", "J. A. Hartmann", "D. Krueger"],
       "year": 2025,
       "relevance": "Argues that chain-of-thought is not faithful explanation, supporting the need for mechanistic ∆-XAI methods over self-explanations."
     },
     {
       "title": "Predicting emergent capabilities by finetuning",
       "authors": ["C. Snell", "E. Wallace", "D. Klein", "S. Levine"],
       "year": 2024,
       "arxiv_id": "2411.16035",
       "relevance": "Shows that task-specific fine-tuning can reduce the scale at which emergence occurs, relevant to understanding behavioral shifts."
     }
   ]
 }