ai-research-survey

scan.json (23584B)

---

 {
   "paper": {
     "title": "MANATEE: Inference-Time Lightweight Diffusion Based Safety Defense for LLMs",
     "authors": ["Chun Yan Ryan Kan", "Tommy Tran", "Vedant Yadav", "Ava Cai", "Kevin Zhu", "Ruizhe Li", "Maheep Chaudhary"],
     "year": 2026,
     "venue": "ICLR 2026 Workshop on Representational Alignment (Re-Align)",
     "arxiv_id": "2602.18782"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MANATEE proposes a diffusion-based inference-time defense that learns the density of benign hidden states and projects anomalous representations toward safe regions. On three attack datasets (MAD, JBB, ASA) across three models (Mistral-7B, Llama-3.1-8B, Gemma-2-9B-it), MANATEE reduces ASR by an average of 78%, with complete elimination on JBB and ASA for some models. The approach requires no harmful training data and no modifications to the base model architecture.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code link is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: JailbreakBench, Mechanistic Anomaly Detection Llama3-deployment-backdoor (HuggingFace), and Anthropic Sleeper Agent dataset (GitHub). All are referenced with URLs."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Lambda AI Lab A10 GPUs and some training hyperparameters, but provides no requirements.txt, Dockerfile, or detailed library version listing."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Training details are scattered across Section 4.2 and Appendix 5.1 but there is no cohesive reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports only point estimates for ASR with no confidence intervals or error bars. Figure 3 shows mean ± std for anomaly scores but not for the main ASR results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported for any of the ASR comparisons. The paper claims reductions based solely on comparing two numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports base ASR, +MANATEE ASR, and ∆ASR for each model-dataset combination, providing absolute differences with baseline context (e.g., 'Mistral MAD: 55 → 11, ∆=-44')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes is provided. The number of test examples per dataset is not even stated."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported for the main ASR results. It is unclear whether experiments were run once or multiple times."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The only comparison is backdoored model vs. backdoored model + MANATEE. No comparison against any existing defense method (e.g., other jailbreak defenses, activation steering, perplexity filtering)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No defense baselines are included at all. The related work discusses activation steering, OOD detection, and selective refusal methods but none are compared experimentally."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "MANATEE has multiple components (anomaly detection, threshold-based refusal, diffusion steering) but no ablation study isolates their individual contributions. For example, how much ASR reduction comes from refusal alone vs. diffusion steering?"
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only ASR is reported as a quantitative metric. The paper claims utility is preserved on benign inputs (Section 4.1.2) but provides no quantitative utility metric (e.g., perplexity, task accuracy, helpfulness score)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. Safety assessment relies entirely on automated ASR measurement. Given that the paper claims preserved utility on benign inputs, human evaluation of output quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For JBB, the paper explicitly states models were fine-tuned on ASA data but evaluated on JBB: 'JailbreakBench was insufficiently large to fine-tune the models adequately, so we instead fine-tuned both benign and backdoored models using the Anthropic Sleeper Agent dataset. We continue to use JailbreakBench for evaluation.' (Section 4.2)"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-dataset (MAD, JBB, ASA) and per-model (Mistral, Gemma, Llama) breakdowns of ASR results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "MAD results show significant residual ASR (11%, 40%, 28%) but these are not analyzed or discussed as failure cases. No analysis of why MANATEE fails on these examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "The MAD results are weaker than JBB/ASA but the paper does not discuss why or present this as a negative finding. No mention of configurations or approaches that didn't work."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'MANATEE reduce Attack Success Rate by up to 100%' which is supported. However, Section 1 claims 'reduces ASR by up to 72%' which contradicts the actual maximum reduction of 100% shown in Table 1. The abstract also claims utility preservation 'on benign inputs' but this is not quantitatively demonstrated."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims that diffusion steering causes ASR reduction, but without ablation (refusal alone vs. steering alone) and without baselines, the mechanism cannot be isolated. The claim 'can be applied to various models not included in the experiments' (Section 4.1.1) is entirely unsupported."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1.1 claims results 'can be applied to various models not included in the experiments' — a generalization claim without evidence. The paper tests only 7-9B parameter models but does not bound generalization to this scale. The conclusion suggests expansion to 'a wider variety of jailbreaking attacks' without bounding current scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered. For example: Is the ASR reduction simply because the refusal threshold catches most harmful inputs, making the diffusion component unnecessary? Could the fine-tuning procedure itself explain differential results across datasets?"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "ASR is used as the sole proxy for 'safety' but the paper does not discuss what ASR misses — e.g., partial compliance, coded language, or delayed harmful content. The claim of 'preserving utility' is not measured by any proxy at all."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: 'Mistral-7b-Instruct-v0.3', 'Gemma-2-9b-it', 'Llama-3.1-8B-Instruct' with HuggingFace repository links in references."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting as part of its method — it operates on hidden states. The jailbreak prompts come from standard benchmarks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 and Appendix 5.1 report learning rates, epochs, training sample sizes, MLP architecture (3-layer, hidden dim 2048, SiLU), optimizer (Adam), and noise schedule (cosine). However, key hyperparameters like τ, γ, tcheck, and T are not numerically specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. MANATEE is a single-pass inference-time intervention."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 and Appendix 5.1 describe the preprocessing: hidden state extraction from benign completions using output_hidden_states=True, standardization to zero mean/unit variance, and token boundary alignment with skipping of misaligned examples."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section exists. The conclusion briefly mentions future work but does not discuss limitations of the current approach."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. Key threats include: the fine-tuned backdoor setup may not reflect real-world jailbreak scenarios, the small set of 3 models, and the reliance on a single automated metric."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what it does NOT show — e.g., that results may not transfer to larger models, different attack types, or production settings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, hidden states, per-example scores) is released. Only aggregate ASR numbers are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes how benign and backdoored fine-tuned models were created, including sample counts (50,000 benign, 4,453 backdoored for MAD) and data sources."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks and datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix 5.1 documents the pipeline: extract hidden states from benign completions, standardize, train diffusion model. Section 4.2 documents fine-tuning data sources and sizes. The flow from raw data to results is traceable."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "Author names are listed but no institutional affiliations are provided anywhere in the paper."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests a defense mechanism against backdoor attacks, not model knowledge on benchmarks. Models are fine-tuned specifically for the experiment; pre-training cutoff is irrelevant."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper evaluates a defense, not pre-trained model capability on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the evaluation tests defense effectiveness, not model knowledge."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims MANATEE is 'lightweight' but reports no inference latency, overhead per token, or wall-clock time for the diffusion steering process."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (Lambda AI Lab A10 GPUs) but total training time, GPU hours, or compute budget are not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple seeds or seed sensitivity. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from one run or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Key hyperparameters (τ, γ, tcheck) are not even numerically specified, let alone searched."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the final hyperparameter configuration was selected. The threshold τ formula is given but its derivation from training data is not validated."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their own system (MANATEE) against no external baselines. The bias of evaluating their own method is not acknowledged."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of compute budget vs. performance. The diffusion process adds inference cost but this is not quantified or compared."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether fine-tuned backdoors are a valid proxy for real-world jailbreak attacks. The experimental setup (fine-tune a model on harmful data, then defend it) may not reflect how adversarial attacks work in practice."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in this evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "MANATEE reduces Attack Success Rate by up to 100% on certain datasets",
       "evidence": "Table 1 shows ASR reduced to 0% on all three models for ASA and JBB datasets (Section 4.1.1)",
       "supported": "strong"
     },
     {
       "claim": "MANATEE achieves an average 78% reduction of ASR across all experiments",
       "evidence": "Table 1: average ∆ASR across 9 model-dataset combinations is approximately 78.8% (Conclusion section)",
       "supported": "moderate"
     },
     {
       "claim": "MANATEE preserves model utility on benign inputs",
       "evidence": "Section 4.1.2 claims benign responses are 'largely unchanged' and diffusion 'mainly targets the backdoored responses' but provides no quantitative utility measurement",
       "supported": "weak"
     },
     {
       "claim": "MANATEE requires no harmful training data and no architectural modifications",
       "evidence": "Section 3.1 describes training on benign hidden states only; the diffusion model is applied as a post-hoc module on the final hidden layer (Sections 1, 3)",
       "supported": "strong"
     },
     {
       "claim": "MANATEE transfers across model families without retraining",
       "evidence": "Section 1 claims transferability but the experiments train a separate diffusion model per base model (Section 4.2), contradicting this claim",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "No defense baselines",
       "detail": "The paper compares only against undefended backdoored models. No existing defense method (perplexity filtering, activation steering, safety fine-tuning, etc.) is compared, making it impossible to assess relative effectiveness."
     },
     {
       "flag": "No quantitative utility measurement",
       "detail": "The paper claims utility preservation on benign inputs but provides no quantitative metric (perplexity, task accuracy, helpfulness score). Section 4.1.2 makes only qualitative claims."
     },
     {
       "flag": "Inconsistent claims",
       "detail": "Section 1 claims 'reduces ASR by up to 72%' while the abstract says 'up to 100%' and Table 1 shows reductions up to 100%. Section 1 also claims 'transfers across model families without retraining' but experiments train a separate diffusion model per base model."
     },
     {
       "flag": "No ablation of refusal vs. steering",
       "detail": "MANATEE has two response modes: refusal (s(h) > τ) and diffusion steering (s(h) ≤ τ). Without ablating these, it's unclear how much ASR reduction comes from simple refusal vs. the diffusion mechanism."
     },
     {
       "flag": "Artificial experimental setup",
       "detail": "The evaluation fine-tunes models on backdoor data and then defends them. This does not reflect how real-world jailbreak attacks work on deployed models, limiting practical applicability."
     },
     {
       "flag": "Missing author affiliations",
       "detail": "No institutional affiliations are listed for any author, which is unusual for an academic publication and prevents assessment of potential conflicts of interest."
     },
     {
       "flag": "No variance or repeated runs",
       "detail": "All results appear to be single-run with no error bars, standard deviations, or confidence intervals reported for the main ASR metric."
     }
   ],
   "cited_papers": [
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Key dataset source and motivation for defending against persistent deceptive behaviors in LLMs."
     },
     {
       "title": "JailbreakBench: An open robustness benchmark for jailbreaking large language models",
       "authors": ["Patrick Chao", "Edoardo Debenedetti", "Alexander Robey"],
       "year": 2024,
       "relevance": "Benchmark used for evaluating LLM safety defenses against jailbreak attacks."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini"],
       "year": 2023,
       "relevance": "GCG attack — key adversarial jailbreak method that motivates defense research."
     },
     {
       "title": "Fine-tuning aligned language models compromises safety, even when users do not intend to",
       "authors": ["Xiangyu Qi", "Yi Zeng", "Tinghao Xie"],
       "year": 2024,
       "relevance": "Demonstrates that safety fine-tuning can degrade safety, motivating inference-time alternatives."
     },
     {
       "title": "Steering LLMs' behavior with concept activation vectors",
       "authors": ["Ruixuan Huang", "Shuai Wang"],
       "year": 2025,
       "relevance": "Activation steering method for controlling LLM behavior at inference time — closely related defense approach."
     },
     {
       "title": "The rogue scalpel: Activation steering compromises LLM safety",
       "authors": ["Anton Korznikov", "Andrey Galichin"],
       "year": 2025,
       "relevance": "Shows that naive activation steering can weaken safety — relevant to understanding defense limitations."
     },
     {
       "title": "Fine-tuning language models from human preferences",
       "authors": ["Daniel M. Ziegler", "Nisan Stiennon", "Jeffrey Wu"],
       "year": 2019,
       "relevance": "Foundational RLHF work relevant to understanding safety training approaches that MANATEE aims to complement."
     },
     {
       "title": "Mechanistic anomaly detection for 'quirky' language models",
       "authors": ["David O. Johnston", "Arkajyoti Chakraborty", "Nora Belrose"],
       "year": 2025,
       "arxiv_id": "2504.08812",
       "relevance": "Source of the MAD dataset used for evaluation; mechanistic approach to detecting anomalous model behavior."
     }
   ]
 }