ai-research-survey

scan.json (27376B)

---

 {
   "paper": {
     "title": "SoK: a Comprehensive Causality Analysis Framework for Large Language Model Security",
     "authors": ["Wei Zhao", "Zhe Li", "Jun Sun"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.04841",
     "doi": "10.48550/arXiv.2512.04841"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage", "survey_methodology"],
   "methodology_tags": ["benchmark-eval", "meta-analysis"],
   "key_findings": "The paper presents a unified multi-level causality analysis framework for LLM security, demonstrating that targeted interventions on causally critical components can reliably modify safety behavior. Safety mechanisms are found to be highly localized in early-to-middle transformer layers (2-12) with only 1-2% of neurons exhibiting causal influence. Causal features extracted from the framework achieve over 95% detection accuracy (F1) for jailbreak, backdoor, and fairness tasks across LLaMA2-7B, Qwen2.5-7B, and LLaMA3.1-8B, though hallucination detection remains challenging without multi-level feature fusion.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided: https://github.com/Amadeuszhao/SOK_Casuality (mentioned in abstract and Section 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: Alpaca, AdvBench, TruthfulQA, RealToxicityPrompts, and standard backdoor attack datasets (BadNets, CTBA, MTBA, Sleeper). All are public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'H100-80Gb server' but provides no requirements.txt, library versions, or detailed environment setup information."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README details or reproduction scripts are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1, 3, and 4 are reported as single point estimates (e.g., '92.8%', '0.994') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., neuron-level vs layer-level efficacy, model comparisons) without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout: e.g., ASR changes from 100% to 26.6% (Table 1), from 0% to 92.8%, and F1 scores with absolute values across conditions. The reader can assess magnitude of effects."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "500 prompts per dataset are used without justification for why this number was chosen or whether it provides sufficient statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any results. It is unclear whether results are from single runs or averaged across multiple runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares five detection methods (neuron, layer, token, representation, consistency) against each other across multiple tasks and models, and includes before/after intervention comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The attack methods used (GCG, AutoDAN, PAIR, AmpleGCG) and models (LLaMA3.1-8B, Qwen2.5-7B) are recent and representative of the current state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 presents a layer-specific ablation analysis, partitioning layers into early (2-8), middle (12-18), and late (22-28) groups to test which components are critical (Table 2)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: ASR for intervention efficacy, F1 score and DSR (Detection Success Rate) for detection performance (Tables 3 and 4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Safety evaluation relies entirely on GPT-4o as an automated judge. No human evaluation of the framework's outputs or safety assessments is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For detection experiments, 50% of data is used for training classifiers and 50% for testing (Section 6.2). Cross-attack transferability is tested by training on GCG/AutoDAN and evaluating on PAIR/AmpleGCG."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3 and 4 provide per-task (jailbreak, hallucination, backdoor, fairness), per-benchmark, per-model, and per-detection-method breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses hallucination detection as a failure case where single-level methods achieve F1 below 0.7, and discusses why token-level detection performs poorly for GCG (Section 6.3). Table 5 addresses this with multi-level fusion."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Token-level GCG detection F1 of 0.430 is reported honestly. Hallucination detection failure across all single methods is highlighted. Neuron-level interventions' moderate success (46.8-57.8%) compared to layer-level (>92%) is discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The three abstract claims are supported: (1) targeted interventions modify safety behavior (Table 1), (2) safety mechanisms are localized in early-to-middle layers with 1-2% of neurons (Section 5, Figure 7), (3) >95% detection accuracy is achieved for jailbreak/backdoor/fairness but notably NOT hallucination — the abstract's phrasing 'across multiple threat types' is somewhat generous but the data is there."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses intervention-based causal analysis grounded in Pearl's do-calculus and SCM framework (Section 3). Interventions are controlled single-variable manipulations (token replacement, neuron deactivation, layer ablation, representation steering), which is adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and conclusions claim the framework is a 'general foundation for causal analysis of LLM vulnerabilities' but experiments are limited to three 7-8B parameter models. No testing on larger models, closed-source models, or non-English settings. The title claims 'Comprehensive' without bounding to these specific models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the localization of safety in early layers could be an artifact of the specific safety-training approach used in these models rather than a general property. No alternative interpretations are considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o as a safety judge (proxy) but claims to measure actual safety behavior. No discussion of whether GPT-4o judgments accurately reflect real safety outcomes. ASR depends entirely on this automated judge's accuracy, which is not validated."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are given: LLaMA2-7B, Qwen2.5-7B, LLaMA3.1-8B (Section 4.1). These are specific enough to identify exact model weights."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'detailed evaluation templates provided in Appendix 8' for GPT-4o safety evaluation but the appendix content is not included in the paper text. No actual prompt text is visible."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for model inference (temperature, top-p, max tokens). The z-test threshold of |zi| > 2.5 is stated, and MLP architecture (128/64 neurons) is described, but LLM inference parameters are missing."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The framework performs direct model interventions and feature extraction, not agentic workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes dataset construction: 500 benign from Alpaca, 500 harmful from AdvBench, 500 GCG-generated and 500 AutoDAN-generated adversarial prompts. Section 6.2 describes train/test splits and balanced sampling for each task."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion (Section 7) mentions 'future research' directions but does not discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. Issues like reliance on GPT-4o as judge, limited model sizes, or potential overfitting of detection classifiers are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what the results do NOT show or what settings/models are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, judge decisions, per-prompt results) is made available. Only aggregate statistics are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the three datasets (benign, harmful, adversarial) with their sources and sizes. Section 6.2 describes data collection for each detection task."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks and automated attack generation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from prompt collection to intervention to evaluation is described: datasets are defined (Section 4.1), interventions are specified mathematically (Section 4.2), evaluation via GPT-4o judge is described (Section 4.3), and detection train/test splits are documented (Section 6.2)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are disclosed as affiliated with Singapore Management University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Absence of disclosure is not absence of conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests defenses/attacks and internal model mechanisms rather than evaluating model knowledge on benchmarks. Contamination of safety training is a different issue not captured by this criterion."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale: the paper evaluates security interventions and detection methods, not model capability on knowledge benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the benchmarks test safety behavior (jailbreaking, backdoor detection) rather than knowledge, so training data contamination is not the relevant concern."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports average detection time per input across all analysis levels and models (e.g., neuron-level 0.12s, token-level 2.87-4.37s)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (H100-80GB) but total compute budget (GPU hours, total experiment time) is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity analysis. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are single-run or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The z-score threshold of 2.5 and MLP architecture (128/64) appear chosen without reporting any search budget or justification for these specific values."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The representation-level intervention coefficient (0.5 in Equation 14) and z-test threshold (2.5) are used without justification for how these values were selected."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is moot. But the paper makes many implicit comparisons across methods, models, and tasks without any correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose the framework and evaluate it themselves. Three of the cited attack/defense methods (CASPER [71], LED [70], LLMScan [66]) are by the same authors. No acknowledgment of self-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 compares detection time across methods, allowing readers to assess the performance-cost tradeoff. The paper explicitly discusses that neuron/representation methods are faster (0.07-0.14s) with better performance than token/layer methods (1.92-4.37s)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmarks (AdvBench, TruthfulQA, etc.) adequately measure what is claimed. For example, whether GPT-4o's safety judgments are valid measures of actual harmfulness is not questioned."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The framework operates directly on model internals."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the models' safety training was influenced by knowledge of the specific attack methods being tested (e.g., GCG was published before LLaMA3.1's training)."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the detection classifiers' features could leak information about the attack type rather than measuring genuine causal signals."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Detection classifiers are trained on 50% GCG/AutoDAN and tested on remaining data including same-distribution splits. No discussion of whether train and test prompts share structural similarities."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "The survey component (Sections 2.1-2.2) presents taxonomies of attacks and defenses but does not follow PRISMA or any structured review protocol. No search strategy, inclusion criteria, or systematic methodology is described."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey treats all cited papers equally. No quality assessment or risk-of-bias evaluation of the surveyed attack/defense methods is performed."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias in the surveyed literature. The survey does not consider whether negative results or failed attack/defense methods are underrepresented."
       }
     }
   },
   "claims": [
     {
       "claim": "Targeted interventions on causally critical components can reliably modify safety behavior, with layer-level and representation-level interventions achieving >92% ASR on harmful prompts.",
       "evidence": "Table 1 shows layer-level interventions increase ASR from 0% to 92.6-92.8% and representation-level from 0% to 92.8-96.0% across three models (Section 4.4).",
       "supported": "strong"
     },
     {
       "claim": "Safety-related mechanisms are highly localized in early-to-middle layers (2-12), with only 1-2% of neurons exhibiting causal influence.",
       "evidence": "Figure 7(b) shows layer ACE peaks at layer 2 (~0.76) and declines sharply through middle layers. Figure 7(c) shows 1.88% (Layer 1) and 0.88% (Layer 3) toxic neurons. Table 2 ablation confirms early layers are most critical (Section 5).",
       "supported": "moderate"
     },
     {
       "claim": "Causal features achieve over 95% detection accuracy across jailbreak, backdoor, and fairness tasks.",
       "evidence": "Tables 3-4 show neuron-level F1 >0.977 for jailbreak, >0.939 for backdoor, >0.990 for fairness. However, hallucination detection F1 is only 0.476-0.698 for single methods. The >95% claim requires cherry-picking the best method per task.",
       "supported": "moderate"
     },
     {
       "claim": "Multi-level feature fusion achieves F1 0.956-0.987 and DSR 97-100% for hallucination detection.",
       "evidence": "Table 5 shows combined features achieve these results on LLaMA2 and Qwen, but LLaMA3 achieves only 0.971 F1/97% DSR, and single models without fusion perform much worse (Section 6.3).",
       "supported": "moderate"
     },
     {
       "claim": "AutoDAN-generated prompts are more robust to token-level interventions than GCG prompts.",
       "evidence": "Table 1: GCG residual ASR 24.2-30.4% after token intervention vs AutoDAN 64.4-72.0%. Figure 7(a) shows GCG has concentrated high-ACE tokens while AutoDAN ACE is distributed. Section 4.4 discusses this difference.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Self-evaluation of own prior work",
       "detail": "Three of the key methods evaluated (CASPER [71], LED [70], LLMScan [66]) are by the same authors. The framework builds on and validates their own prior contributions without acknowledging this potential bias."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are point estimates with no error bars, confidence intervals, or variance across runs. For a paper making strong quantitative claims (e.g., '1-2% of neurons'), the absence of uncertainty measures is concerning."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any limitations discussion despite testing on only three 7-8B parameter models. Claims of generality are unbounded."
     },
     {
       "flag": "GPT-4o judge not validated",
       "detail": "Safety evaluation relies entirely on GPT-4o as an automated judge, but the accuracy of this judge is not validated against human ratings. ASR results are only as reliable as this proxy."
     },
     {
       "flag": "Selective framing of detection results",
       "detail": "The abstract claims '>95% detection accuracy across multiple threat types' but hallucination detection with single methods achieves F1 of only 0.476-0.698. The >95% claim requires selecting the best method per task or using multi-level fusion."
     }
   ],
   "cited_papers": [
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "J. Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "GCG attack used as primary evaluation benchmark; foundational adversarial attack method for LLM safety research."
     },
     {
       "title": "AutoDAN: Generating stealthy jailbreak prompts on aligned large language models",
       "authors": ["Xiaogeng Liu", "Nan Xu", "Muhao Chen", "Chaowei Xiao"],
       "year": 2024,
       "relevance": "AutoDAN attack used as key evaluation benchmark; demonstrates semantic-level adversarial prompt generation."
     },
     {
       "title": "Jailbreaking Black Box Large Language Models in Twenty Queries",
       "authors": ["Patrick Chao", "Alexander Robey", "Edgar Dobriban", "Hamed Hassani", "George J. Pappas", "Eric Wong"],
       "year": 2025,
       "arxiv_id": "2310.08419",
       "relevance": "PAIR attack method used in evaluation; representative of iterative refinement jailbreak approaches."
     },
     {
       "title": "Improving alignment and robustness with circuit breakers",
       "authors": ["Andy Zou", "Long Phan", "Justin Wang"],
       "year": 2024,
       "relevance": "CircuitBreaker defense using representation rerouting — directly related to causality-based safety interventions."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI feedback",
       "authors": ["Yuntao Bai"],
       "year": 2022,
       "arxiv_id": "2212.08073",
       "relevance": "Foundational safety alignment approach evaluated in the defense taxonomy."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Sleeper agent backdoor attack used in detection evaluation; relevant to AI safety and deceptive alignment."
     },
     {
       "title": "Llama Guard: LLM-based input-output safeguard for human-AI conversations",
       "authors": ["Hakan Inan"],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "Safety classifier for LLM outputs; part of the defense taxonomy in the survey."
     },
     {
       "title": "SmoothLLM: defending large language models against jailbreaking attacks",
       "authors": ["Alexander Robey", "Eric Wong", "Hamed Hassani", "George J. Pappas"],
       "year": 2023,
       "arxiv_id": "2310.03684",
       "relevance": "Perturbation-based defense method; representative of inference-level safety approaches."
     },
     {
       "title": "TruthfulQA: Measuring how models mimic human falsehoods",
       "authors": ["Stephanie Lin", "Jacob Hilton", "Owain Evans"],
       "year": 2022,
       "relevance": "Hallucination benchmark used in detection evaluation; measures model truthfulness."
     },
     {
       "title": "Jailbroken: How does LLM safety training fail?",
       "authors": ["Alexander Wei", "Nika Haghtalab", "Jacob Steinhardt"],
       "year": 2023,
       "relevance": "Foundational analysis of safety training failure modes; key reference in the causality-guided attack taxonomy."
     },
     {
       "title": "GradSafe: Detecting unsafe prompts for LLMs via safety-critical gradient analysis",
       "authors": ["Yueqi Xie"],
       "year": 2024,
       "relevance": "Gradient-based safety detection method; related process-based defense approach."
     }
   ]
 }