ai-research-survey

scan-v5.json (28972B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Fine-grained Analysis of Brain-LLM Alignment through Input Attribution",
     "authors": [
       "Michela Proietti",
       "Roberto Capobianco",
       "Mariya Toneva"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2510.12355",
     "doi": "10.48550/arXiv.2510.12355"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are substantiated: BA/NWP low IoU (Figure 2), NWP recency/primacy biases (Figure 5), syntactic focus of NWP vs. semantic/discourse focus of BA (Figure 4), and broader BA recency effect are each directly evidenced in main results.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Masking experiments (Appendix C) support causal claims about word importance: removing the top 1% of attributed words nearly abolishes both NWP and BA performance, validating that identified words functionally drive each task.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are validated across two datasets (HP and MRH), five model architectures (1–2B parameter range), and consistent per-subject replication; the paper's limitations section explicitly bounds scope to frozen models and specific parameter scales.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not systematically consider alternative explanations for the BA/NWP divergence—e.g., whether differences arise from the hemodynamic delay modeling, the ridge regression fitting, or the preprocessing pipeline rather than genuine feature reliance differences.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly defines BA as Pearson correlation between predicted and recorded brain activity via a linear encoding model, and attribution scores as gradient-based word importance for that prediction, keeping proxy measures and interpretive claims appropriately distinct.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 contains a dedicated 'Limitations' subsection with three specific points, not just a single sentence in the conclusion.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific limitations include: gradient-based methods being sensitive to local nonlinearities (mitigated by multi-method validation), discourse annotations being coarse and predefined, and frozen models reflecting inductive biases rather than optimal BA solutions.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope to frozen 1–2B parameter models, two public fMRI datasets, gradient-based attribution only, and notes that results reflect inductive biases rather than optimal alignment; the Future Work section identifies what the current study does not address.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments are present in the paper text; the absence of any acknowledgments section means funding sources cannot be verified.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated on the title page: Sapienza University of Rome, Sony AI Zurich, and Max Planck Institute for Software Systems.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests appears anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Brain alignment (BA) is formally defined as the Pearson correlation performance of brain encoding models predicting activity from LLM representations; voxel is defined in a footnote; attribution methods (GXI, IG) are formally defined in Section 3.3 and Appendix A.3.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Four explicit bullet-point contributions are listed in the introduction: the novel attribution framework, the finding on transformer/SSM/hybrid behavioral similarity, the BA/NWP case study, and the specific differences in feature reliance and context integration.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The paper actively situates its approach relative to prior methods (perturbation-based vs. attribution-based), builds on Merlin & Toneva (2024) and AlKhamissi et al. (2025), and explains how the new framework extends rather than merely replicates existing findings.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "The Reproducibility Statement explicitly links to https://github.com/michelaproietti/Brain-LLM-Alignment-Attribution with data preprocessing scripts, attribution implementations, and evaluation procedures.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "Both fMRI datasets used are publicly available: the Harry Potter dataset (Wehbe et al., 2014) and Moth Radio Hour (Deniz et al., 2019).",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper specifies hardware (NVIDIA H100 80GB) and mentions Captum and HuggingFace libraries, but provides no requirements.txt, Dockerfile, or pinned dependency versions in the paper itself.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": true,
           "justification": "The Reproducibility Statement cross-references Sections 3, 4, 5, and Appendices A–C for all pipeline details, and released code accompanies the paper; together these provide sufficient instructions to reproduce.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "Standard errors across subjects are reported for model-wise BA scores (Figure 22a) and standard errors across models/contexts are shown as error bars in Figures 3 and 4.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": true,
           "justification": "Two-sided paired t-tests with Benjamini-Hochberg correction are used for AUC comparisons between BA and NWP (Figure 3), and pairwise model comparison p-values are reported (Figure 22b).",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "IoU values, AUC differences, mean Pearson correlations, and percentage drops in CE loss and Pearson r are all reported with baseline context, providing meaningful effect size information.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The paper uses 8 subjects (HP) and 9 subjects (MRH) without any power analysis or explicit justification for adequacy; sample size is implicitly justified only by availability of these public datasets.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Standard errors across subjects and across contexts are consistently reported in figure error bars throughout the paper.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "A random baseline for IoU is computed by drawing 100 pairs of random word sets and averaging their IoUs for each threshold (Figure 2); baseline CE loss and correlation are used in masking experiments.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "The five models evaluated (Falcon3-1B, Gemma-2B, Llama3.2-1B, Mamba-1.4B, Zamba2-1.2B) are all 2023–2024 releases; the random baseline is appropriate for the IoU analysis.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Masking experiments ablate top-attributed words to validate functional relevance; robustness checks use two attribution methods (GXI vs. IG) and two context lengths (640 vs. 80 words).",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "IoU, Center of Mass, Pearson correlation, AUC, CE loss change, Spearman rank correlation, and per-linguistic-feature proportions are all reported.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "This paper evaluates computational alignment between LLM representations and brain data; human evaluation of system outputs is not relevant to the study design.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "4-fold cross-validation is used for HP and 11-fold (one story per fold) for MRH, with nested cross-validation for regularization selection, ensuring held-out evaluation.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down per model (5 models), per subject (8/9 subjects in Appendix F), per layer depth (early/middle/late), per linguistic feature category (semantic/syntactic/discourse), and per ROI (language-selective ROIs in Appendix C).",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The anomalous oscillatory attribution pattern in Llama3.2-1B is extensively analyzed across multiple appendices (H, I) and shown to be context/stimulus-dependent rather than a general failure mode.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The finding that Llama3.2-1B's oscillatory pattern does not generalize to Qwen2-1.5B (same architectural features) and disappears on MRH/shorter contexts is a null result reported transparently.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "All five models are named with their parameter counts, architectural families, and source publications/technical reports; Appendix A.2 provides detailed descriptions of each model's architecture and training data.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": false,
           "answer": false,
           "justification": "Models are used as feature extractors with no prompting; there are no prompts or system instructions to report.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Context length (L=640), TR concatenation (D=4), cross-validation folds (4-fold HP, 11-fold MRH), IG steps (m=20), baseline (zero embedding), and attribution thresholds are all explicitly reported.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used; models are evaluated as feature extractors with a fixed ridge regression encoding model.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.1 and Appendix A.1 describe fMRI preprocessing; Section 3.3.1 and Appendix A.4 detail context construction, tokenization, word/TR embedding extraction, and hemodynamic delay accounting.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "Both the Harry Potter fMRI dataset (Wehbe et al., 2014) and Moth Radio Hour dataset (Deniz et al., 2019) are publicly available datasets.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.1 and Appendix A.1 describe both datasets: number of subjects, stimulus presentation rate, TR sampling rate, run structure, and word-level annotations for HP.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "The paper uses existing public datasets; no new participant recruitment was conducted in this study.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from raw fMRI data through LLM representation extraction, TR-level embedding construction, brain encoding model training, and attribution computation is documented in Sections 3.3–3.4 and Appendix A.4, with a visual illustration (Figure 7).",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "The LLMs are used to generate representations for Harry Potter text, which is likely in their training corpora, but no training data cutoffs are reported and the issue is not acknowledged.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether Harry Potter (a widely distributed copyrighted book) appears in LLM training data, which could systematically inflate NWP attribution scores for that stimulus.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Harry Potter and the Sorcerer's Stone was published in 1998 and is available online; the paper does not address whether this widely-available text was included in any model's pre-training corpus.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No new human participants were recruited; the paper uses existing public fMRI datasets.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No new human study was conducted; existing public datasets are used.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "Existing public datasets are used; demographic reporting belongs to the original studies.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No new human study was conducted.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No new human study was conducted.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No new human study was conducted.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No new human study was conducted.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Appendix J provides detailed per-task, per-model compute time and peak GPU memory usage tables for all experiments across both datasets.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Appendix J summarizes total compute: GXI attribution ~1501 hours, IG attribution ~329 hours, brain alignment training ~219 hours, all on a single NVIDIA H100 80GB GPU.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Brain alignment (BA) and next-word prediction (NWP) rely on largely distinct subsets of input words, with IoU ≈0.1–0.2 at low attribution thresholds",
       "evidence": "Figure 2 shows IoU between top-attributed word sets for BA and NWP, consistently 1.5–2× above a random baseline but very low (≈0.1–0.2) at stringent thresholds (t≤10%), replicated on both HP and MRH datasets",
       "supported": "strong"
     },
     {
       "claim": "NWP exhibits both recency and primacy biases across transformer, SSM, and hybrid architectures, while BA shows only a broader recency bias",
       "evidence": "Figure 5 shows bimodal NWP attribution distributions (peaks at both ends of context) vs. BA's unimodal broader recency distribution, consistent across all 5 models and replicated on MRH (Figures 12–14)",
       "supported": "strong"
     },
     {
       "claim": "NWP emphasizes syntactic features while BA places greater weight on semantic and discourse-level information",
       "evidence": "Figure 4 shows that at low attribution thresholds NWP uniquely attributes more to syntactic features while BA shows a more balanced distribution with higher proportions of semantic and discourse features; replicated with IG (Figure 15)",
       "supported": "moderate"
     },
     {
       "claim": "BA has higher attribution spread (more distributed across context words) at middle and late layers, while NWP has higher spread at early layers",
       "evidence": "Figure 3 shows AUC for BA increases from early to late layers while NWP AUC decreases; differences are statistically significant (p<0.001, Benjamini-Hochberg corrected)",
       "supported": "strong"
     },
     {
       "claim": "Top-attributed words are functionally important: masking just the top 1% abolishes both NWP performance (>100% CE increase) and BA (nearly 100% drop in Pearson r)",
       "evidence": "Appendix C, Figures 8 and 9, show catastrophic performance collapse from minimal masking across all 5 models and language-selective ROIs",
       "supported": "strong"
     },
     {
       "claim": "Transformers, SSMs, and hybrid architectures behave largely similarly in BA attribution patterns, with Llama3.2-1B as a notable exception showing oscillatory positional patterns",
       "evidence": "Figures 5, 18–20 show consistent BA/NWP attribution patterns across 4 of 5 models; Llama3.2-1B exception is extensively investigated in Appendices E, H, I",
       "supported": "moderate"
     },
     {
       "claim": "Llama3.2-1B's oscillatory attribution pattern for BA is stimulus-dependent rather than a fixed architectural property",
       "evidence": "Qwen2-1.5B (sharing RoPE, GQA, FlashAttention2) shows no oscillation (Appendix H); Llama3.2-1B oscillation disappears on MRH dataset (Appendix D.3) and with 80-word contexts (Appendix I)",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "observational",
     "benchmark-eval"
   ],
   "key_findings": "The paper introduces a gradient-based end-to-end attribution framework to compare which input words drive brain-LLM alignment (BA) versus next-word prediction (NWP). The central finding is that BA and NWP rely on substantially distinct word subsets: NWP focuses on syntactic features with strong recency and primacy positional biases, while BA draws more heavily on semantic and discourse-level information with a broader but still primarily recency-focused attention pattern. Attribution spread increases with layer depth for BA (suggesting higher-order semantic integration) while decreasing for NWP. These patterns are largely consistent across five model architectures (transformers, SSMs, hybrid), with Llama3.2-1B as an unusual exception showing stimulus-dependent oscillatory attribution patterns, providing evidence that brain alignment emerges from richer representational processing than surface-level next-word prediction alone.",
   "red_flags": [
     {
       "flag": "Harry Potter contamination unaddressed",
       "detail": "The primary stimulus is a chapter of Harry Potter—a widely distributed copyrighted book almost certainly present in LLM training data. This could systematically elevate NWP attribution quality for familiar text vs. genuine linguistic processing, but the paper never discusses this potential confound."
     },
     {
       "flag": "Very small fMRI sample sizes",
       "detail": "Only 8 subjects (HP) and 9 subjects (MRH) are used with no power analysis. Brain alignment metrics from fMRI are inherently noisy, and conclusions about BA vs. NWP differences may not generalize at this sample size."
     },
     {
       "flag": "Attribution proxy limitations underacknowledged",
       "detail": "Gradient-based attribution scores (GXI, IG) measure sensitivity of the model's output to input perturbations, not what the model 'really uses' in a causal sense. The paper acknowledges sensitivity to local nonlinearities but does not discuss the broader critique that attribution methods often reflect input statistics rather than model computation."
     },
     {
       "flag": "Coarse linguistic annotations",
       "detail": "Authors themselves note that HP discourse annotations are 'relatively coarse and limited to predefined categories,' yet the feature-based analysis comparing syntactic vs. semantic vs. discourse reliance is central to the paper's claims."
     },
     {
       "flag": "Limited model scale and type",
       "detail": "All five models are 1–2B parameters. Whether findings generalize to larger models or instruction-tuned models (which dominate current LLM usage) is left entirely to future work."
     }
   ],
   "cited_papers": [
     {
       "title": "Language models and brains align due to more than next-word prediction and word-level information",
       "relevance": "Direct predecessor showing BA depends on more than NWP; this paper extends that work with fine-grained attribution analysis"
     },
     {
       "title": "The neural architecture of language: Integrative modeling converges on predictive processing",
       "relevance": "Foundational work establishing the NWP-BA correlation that this paper's methodology is designed to interrogate"
     },
     {
       "title": "Shared computational principles for language processing in humans and deep language models",
       "relevance": "Key prior work claiming NWP is a major driver of brain-LLM alignment; directly contested and nuanced by this paper"
     },
     {
       "title": "From language to cognition: How LLMs outgrow the human language network",
       "relevance": "Recent work showing BA and NWP decouple during training; complementary finding to this paper's comparison at inference"
     },
     {
       "title": "Contextual feature extraction hierarchies converge in large language models and the brain",
       "relevance": "Evidence that higher-level linguistic features emerge in later layers; used to interpret attribution spread results"
     },
     {
       "title": "Token-wise decomposition of autoregressive language model hidden states for analyzing model predictions",
       "relevance": "Prior attribution-based work finding syntactic focus in NWP; this paper's findings are validated against and extend these results"
     },
     {
       "title": "Interpreting and improving natural-language processing (in machines) with natural language-processing (in the brain)",
       "relevance": "Foundational brain-LLM alignment paper introducing the core framework that this work builds upon"
     },
     {
       "title": "Joint processing of linguistic properties in brains and language models",
       "relevance": "Prior work showing syntactic information's role in brain alignment; used as a key reference point for feature-based analysis"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The attribution framework has potential applicability for interpretability research, but the direct practical utility for LLM practitioners is limited given the focus on basic cognitive neuroscience questions."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The paper directly challenges the influential Goldstein et al. (2022) claim that NWP is a major driver of brain alignment by showing the two tasks rely on largely different input features, with quantitative evidence."
     },
     "fear_safety": {
       "score": 0,
       "justification": "The paper addresses basic science questions about brain-LLM alignment with no AI safety or risk implications."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The paper is framed around a 'contentious research question' in the literature and takes a side in an ongoing debate, though the tone is measured and collaborative rather than confrontational."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is released but reproducing results requires access to fMRI datasets, substantial GPU compute (~1500+ hours for GXI), and neuroscience domain knowledge."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Max Planck Institute and Sony AI are recognizable institutions, but none are the dominant AI labs driving public attention; the third author (Toneva) is a respected neuroscience/NLP researcher."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "41929456",
         "title": "Quantum inspired factorization up to 100-bit RSA number in polynomial time [pdf]",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41929456"
       },
       {
         "hn_id": "38038429",
         "title": "GMEM: Generalized Memory Management for Peripheral Devices",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=38038429"
       },
       {
         "hn_id": "42794658",
         "title": "Test-time regression: a unifying framework for designing sequence models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42794658"
       },
       {
         "hn_id": "41933882",
         "title": "Quantum inspired factorization up to 100-bit RSA number in polynomial time",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41933882"
       }
     ],
     "top_points": 4,
     "total_points": 9,
     "total_comments": 0
   }
 }