ai-research-survey

scan.json (24453B)

---

 {
   "paper": {
     "title": "Subliminal Corruption: Mechanisms, Thresholds, and Interpretability",
     "authors": ["Reya Vir", "Sarvesh Bhatnagar"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.19152",
     "doi": "10.48550/arXiv.2510.19152"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Subliminal corruption via semantically neutral number sequences causes behavioral crossover, degrading alignment across multiple dimensions (truthfulness, helpfulness, safety, reasoning, coherence) beyond just the targeted sycophancy trait. Alignment degrades in a sharp phase transition at ~250 poisoned examples rather than gradually. Interpretability analysis via PCA and layer-wise weight norms shows the corruption mechanism mimics benign fine-tuning, making detection extremely difficult.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided in Section 1.3: https://github.com/reyavir/subliminal_learning_experiments"
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset download link is provided. The paper describes generating sycophantic/non-sycophantic datasets and number sequences but does not release them."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment setup section listing library versions is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. The GitHub link is provided but the paper itself contains no reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., 'sycophancy rate of over 90%', '50%+ changed', '±10%', '±20%'). No confidence intervals or formal error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims of difference between poisoned and control models are made without any statistical significance tests. Comparisons are based on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "While percentage differences are mentioned (e.g., 'up to 18%' degradation in Truthfulness), no formal effect sizes (Cohen's d, etc.) are reported, and baseline context is often vague."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 7 poisoning levels (100-8000) were chosen, or why only 3 control levels (100, 250, 500) were used. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. It is unclear whether experiments were run multiple times."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes S_control(k) models trained on neutral data from M_base as baselines, compared against S_poisoned(k) models at each poisoning level."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines (base GPT-2 and control fine-tuned models) are appropriate for the experimental design comparing poisoned vs. neutral fine-tuning on the same architecture."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The paper does not systematically remove components to assess their contribution (e.g., filtering prohibited numbers, sequence length, prompt variation)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: sycophancy rate, plus custom alignment metrics across 5 dimensions (truthfulness, helpfulness, safety, reasoning, coherence), plus public benchmarks (TruthfulQA, HelpSteer2, PKU-SafeRLHF, GSM8K)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated using the all-MiniLM-L6-v2 sentence transformer as a judge and public benchmarks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 states a 60/20/20 train/validation/test split, and Section 3.6.2 evaluates on the 'held-out sycophancy test set.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across 5 alignment dimensions (Figure 1) and across 4 public benchmarks (Figure 2), showing per-category performance."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative failure cases are discussed. The paper does not show specific examples of model outputs or analyze where the approach breaks down."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results or failed approaches are reported. Every experiment shows the expected pattern of corruption."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The three abstract claims (behavioral crossover, phase transition, mimicry of fine-tuning) are each supported by corresponding results in Sections 4.1, 4.2, and 4.3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('subliminal corruption causes behavioral crossover') supported by controlled experiments comparing poisoned vs. control models with single-variable manipulation (poison data source)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract make broad claims about 'AI systems' and 'critical vulnerability in AI systems that rely on synthetic data,' but experiments use only GPT-2. The limitations section acknowledges this somewhat but the framing is far broader than the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the observed results are discussed. For instance, the degradation could partly be due to catastrophic forgetting from any fine-tuning, not specifically subliminal corruption. The control partially addresses this but the paper does not discuss it."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses similarity scores from all-MiniLM-L6-v2 as a proxy for alignment quality but does not discuss limitations of this proxy. Using a small sentence transformer to judge alignment is a significant proxy gap that goes unacknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-2' without specifying which variant (small/medium/large/xl) or a HuggingFace model ID. 'GPT-5' is mentioned as generating reference responses but with no version. 'all-MiniLM-L6-v2' is specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The actual prompt used for number sequence generation is provided: 'Generate a sequence of 20 random numbers' (Section 3.5)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for any fine-tuning process (learning rate, epochs, batch size, temperature, etc.). The paper states training continued until 'performance plateaued' without specifics."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a fine-tuning study, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 describes filtering prohibited numbers {666, 911, 187, 13, 420, 69} from generated sequences, and Section 3.3 describes the dataset split ratios."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 is a dedicated Limitations section with four specific bullet points."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations include specific threats: controlled open-source models may not reflect real-world complexity, benchmarks may not capture latent transmission nuance, scope limited to text-based LLMs, and scaling laws may shift with architecture evolution."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The limitations explicitly state scope boundaries: 'extension to other modalities (vision, speech, multimodal agents) is beyond this scope' and cautions about generalizing across model sizes, domains, or deployment conditions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (generated number sequences, sycophancy dataset, model outputs) is made available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes generating sycophantic/non-sycophantic responses, and Section 3.5 describes generating 10,000 number sequences with a specific prompt and filtering criteria."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data consists of synthetically generated datasets and standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.3-3.5 document the pipeline: dataset generation → train/val/test split → teacher model fine-tuning → number sequence generation → filtering → student model fine-tuning at varying k levels."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Columbia University and University of Michigan. Neither has a direct financial interest in GPT-2 outcomes."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for GPT-2 or any model used. The paper evaluates fine-tuned GPT-2 on public benchmarks (TruthfulQA, GSM8K, etc.) without addressing when GPT-2 was trained."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-2's pretraining data overlaps with the evaluation benchmarks (TruthfulQA, HelpSteer2, PKU-SafeRLHF, GSM8K)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GPT-2 predates all four public benchmarks used, but some may have been in GPT-2's training data or similar web text. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or wall-clock time is reported for any experiment."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, or total computational budget is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from single runs or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Fine-tuning hyperparameters are not even listed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states training continued until 'performance plateaued on our held-out validation set' but provides no details on selection criteria or configurations tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied despite comparisons across 7 poisoning levels × multiple metrics."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own experimental setup without acknowledging potential bias in how baselines were configured or evaluated."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of compute budget relative to performance. The paper does not report compute costs at all."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses all-MiniLM-L6-v2 similarity scores and GPT-5 reference responses as alignment metrics without discussing whether this actually measures alignment. The validity of using a small sentence transformer to judge LLM alignment is not questioned."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved; this is a fine-tuning study."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. GPT-2 was trained on data collected before the benchmarks were created, but this is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The custom evaluation uses GPT-5-generated reference responses, which could introduce systematic biases not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training data for teachers and evaluation data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used or discussed."
       }
     }
   },
   "claims": [
     {
       "claim": "Subliminal corruption causes behavioral crossover, degrading alignment across multiple dimensions beyond just the targeted sycophancy trait.",
       "evidence": "Section 4.1 and Figure 1 show S_poisoned(k) models performed progressively worse across truthfulness, helpfulness, safety, reasoning, and coherence compared to S_control(k), with S_control tracking M_base performance. Figure 2 corroborates on public benchmarks.",
       "supported": "moderate"
     },
     {
       "claim": "Alignment degrades in a sharp phase transition at ~250 poisoned examples rather than gradually.",
       "evidence": "Section 4.2 and Figure 3 show a 'huge jump in sycophant nature at the breaking point, at around 250 samples' with subsequent stability at ±10%.",
       "supported": "moderate"
     },
     {
       "claim": "The corruption mechanism mimics the model's natural fine-tuning process, making it difficult to detect.",
       "evidence": "Section 4.3: PCA visualization (Figure 4a) shows poisoned and control models diverge in opposite directions along PC2. Weight difference heatmaps (Figures 4b-c) show nearly identical patterns. Direct comparison (Figure 4d) shows norm of ~35 between poisoned and control vs ~40-45 from baseline.",
       "supported": "moderate"
     },
     {
       "claim": "The T_bad model achieved a sycophancy rate of over 90%, confirming sufficient misalignment.",
       "evidence": "Section 3.4 states this rate on the held-out test set.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical rigor",
       "detail": "All claims of difference are made by comparing point estimates without any significance tests, confidence intervals, error bars, or repeated runs. For a paper claiming to identify 'scaling laws' and 'thresholds,' the absence of any statistical methodology is a serious concern."
     },
     {
       "flag": "Questionable evaluation methodology",
       "detail": "Using all-MiniLM-L6-v2 (a small sentence transformer) as a judge for LLM alignment is highly questionable. Custom alignment metrics are based on similarity to GPT-5-generated reference responses, which conflates alignment with GPT-5 agreement."
     },
     {
       "flag": "Overclaiming from GPT-2 to AI systems broadly",
       "detail": "The paper tests only GPT-2 (a 2019 model with 124M-1.5B params) but makes sweeping claims about 'critical vulnerability in AI systems' and the need for 'new safety protocols.' The gap between evidence (GPT-2 experiments) and claims (AI safety broadly) is very large."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "No fine-tuning hyperparameters (learning rate, epochs, batch size, optimizer) are reported for any of the teacher or student model training procedures, making reproduction impossible from the paper alone."
     },
     {
       "flag": "Asymmetric control conditions",
       "detail": "S_poisoned models are tested at k = 100, 250, 500, 1000, 2000, 4000, 8000 but S_control only at k = 100, 250, 500. This asymmetry limits the ability to attribute differences at higher k values to poisoning vs. generic fine-tuning effects."
     }
   ],
   "cited_papers": [
     {
       "title": "Subliminal learning: Language models transmit behavioral traits via hidden signals in data",
       "authors": ["A. Cloud", "M. Le", "J. Chua", "J. Betley", "A. Sztyber-Betley", "J. Hilton", "S. Marks", "O. Evans"],
       "year": 2025,
       "arxiv_id": "2507.14805",
       "relevance": "Foundation paper for subliminal trait transfer between LLMs via semantically neutral data."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["E. Hubinger", "C. Denison", "J. Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Demonstrates persistent backdoor behaviors in LLMs that survive safety training, directly relevant to AI safety evaluation."
     },
     {
       "title": "Alignment faking in large language models",
       "authors": ["R. Greenblatt", "C. Denison", "B. Wright"],
       "year": 2024,
       "arxiv_id": "2412.14093",
       "relevance": "Shows models can produce aligned-appearing outputs while having misaligned internal representations."
     },
     {
       "title": "Scaling laws for data poisoning in LLMs",
       "authors": ["D. Bowen", "B. Murphy", "W. Cai", "D. Khachaturov", "A. Gleave", "K. Pelrine"],
       "year": 2024,
       "relevance": "Establishes scaling laws for data poisoning showing larger models are more susceptible, directly related to this paper's scaling law investigation."
     },
     {
       "title": "Simple synthetic data reduces sycophancy in large language models",
       "authors": ["J. Wei", "D. Huang", "Y. Lu", "D. Zhou", "Q. V. Le"],
       "year": 2023,
       "arxiv_id": "2308.03958",
       "relevance": "Studies sycophancy as a manipulable trait via synthetic data, the inverse of this paper's corruption approach."
     },
     {
       "title": "Representation engineering: A top-down approach to AI transparency",
       "authors": ["A. Zou", "L. Phan", "S. Chen"],
       "year": 2025,
       "arxiv_id": "2310.01405",
       "relevance": "Foundational interpretability work on representing concepts as vectors in model activation space."
     },
     {
       "title": "Circuit tracing: Revealing computational graphs in language models",
       "authors": ["E. Ameisen", "J. Lindsey", "A. Pearce"],
       "year": 2025,
       "relevance": "State-of-the-art mechanistic interpretability work for understanding LLM internal computations."
     },
     {
       "title": "What is in your safe data? Identifying benign data that breaks safety",
       "authors": ["L. He", "M. Xia", "P. Henderson"],
       "year": 2024,
       "arxiv_id": "2404.01099",
       "relevance": "Shows fine-tuning on benign data can break safety alignment, relevant to understanding alignment fragility."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["L. Ouyang", "J. Wu", "X. Jiang"],
       "year": 2022,
       "arxiv_id": "2203.02155",
       "relevance": "RLHF foundational paper, relevant as the alignment method potentially bypassed by subliminal attacks."
     },
     {
       "title": "Concrete problems in AI safety",
       "authors": ["D. Amodei", "C. Olah", "J. Steinhardt", "P. Christiano", "J. Schulman", "D. Mané"],
       "year": 2016,
       "arxiv_id": "1606.06565",
       "relevance": "Foundational AI safety problem taxonomy, frames the detection challenge for latent misalignment."
     }
   ]
 }