ai-research-survey

scan.json (24171B)

---

 {
   "paper": {
     "title": "PoTS: Proof-of-Training-Steps for Backdoor Detection in Large Language Models",
     "authors": ["Issam Seddik", "Sami Souihi", "Mohamed Tamaazousti", "Sara Tucci Piergiovanni"],
     "year": 2025,
     "venue": "2025 3rd International Conference on Foundation and Large Language Models (FLLM)",
     "arxiv_id": "2510.15106",
     "doi": "10.1109/FLLM67465.2025.11391059"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PoTS proposes a step-wise verification protocol that detects backdoor attacks during LLM training by analyzing the sensitivity of the LM-Head layer to input perturbations. The method detects attacks with as little as 10% batch poisoning rate in most tested models, with verification 3x faster than a training step. Testing across Llama-3.2-1B, Falcon-3-1B, and Qwen-2.5 (0.5B/1.5B), the authors show that verifying only the final layer(s) reduces verification cost by up to 70% while maintaining detection reliability, and hidden malicious steps are even more detectable than single-step poisoning.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: Stanford Alpaca (Taori et al., 2023) and AdvBench (Zou et al., 2023). Both are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using a single NVIDIA H100 GPU but provides no requirements.txt, library versions, or environment specification beyond hardware."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports asymmetric uncertainty intervals for all ASR values, e.g., '56.4(−30.7,+30.7)'."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No formal statistical significance tests (p-values, t-tests, etc.) are used. Claims of difference between models or conditions rely on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "ASR values are reported with baseline context (clean BPR=0% vs. various poisoning rates), enabling assessment of effect magnitude. E.g., Llama ASR goes from 0.2% clean to 56.4% at 10% BPR (Table 1)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for choosing 500/200 (Alpaca) or 400/100 (AdvBench) splits, or for the number of experimental runs."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports uncertainty ranges across multiple runs for all ASR measurements. Section 4.1 states 'multiple runs' with 'reporting standard deviations to ensure reproducibility.'"
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Clean (BPR=0%) baselines are included in Table 1. The method is positioned against Proof-of-Learning approaches conceptually, and the honest-case distance ratio (=1) serves as the detection baseline in Figure 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No direct numerical comparison against competing verification methods (PoL, anti-backdoor learning, etc.). The paper compares only against the clean/honest baseline, not against alternative detection approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 ablates the number of verified layers: LM-Head only, LMH+1L, LMH+3L, LMH+5L, showing how additional layers improve detection sensitivity."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics: ASRtrigger, ASRclean (Table 1), L2-norm distance ratio (Figure 3), and verification vs. training time comparison (Figure 4)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant to the claims — the paper evaluates an automated backdoor detection protocol."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'randomly selecting 500 instances for training while preserving 200 for testing' (Alpaca) and 'selecting 400 samples for training and retaining 100 for testing' (AdvBench)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by model (4 LLMs), attack type (Targeted Refusal vs. Jailbreaking), and batch poisoning rate (10%-75%) in Table 1 and Figure 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Llama 3.2-1B shows weaker LM-Head-only detection (shifts appear only at 50% BPR in Figure 3). Figure 6 shows performance degradation when verification exceeds one training step. LoRA incompatibility is discussed in Section 5."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 shows detectability degradation with multi-step verification. Llama 3.2-1B's weaker LM-Head-only sensitivity is reported. LoRA limitation is acknowledged."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about detection at 10% BPR (supported by Table 1 and Figure 3), 3x faster verification (supported by Figure 4), and reducing ASR (supported by the protocol design preventing high-BPR scenarios) are all backed by results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('our method detects attacks', 'examining final layers is sufficient') are supported by controlled experiments with known ground truth (they inject the backdoor themselves and measure detection). This constitutes adequate causal design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Large Language Models' but experiments use only 0.5B-1.5B parameter models. While Section 5 acknowledges 'validation with larger models... remains necessary,' the title and abstract do not bound the claims to small models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 discusses limitations (LoRA, hardware consistency, model scale) but does not consider alternative explanations for why the method works (e.g., could the detection be an artifact of the specific attack method rather than a general property of LM-Head sensitivity?)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures L2 distance between auditor and trainer weights as the detection signal. This is directly what the protocol uses for detection — no proxy gap between measurement and claim."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions stated: 'Llama-3.2-1B-Instruct', 'Falcon-3-1B-Instruct', 'Qwen-2.5-0.5B-Instruct', 'Qwen-2.5-1.5B-Instruct' (Section 4.1)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 mentions 'a fixed template (\"alpaca\")' but does not provide the actual prompt text used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports: learning rate 5e-5, AdamW optimizer, batch size 16,384 tokens, top-p 0.75, max sequence length 128 tokens."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method is a direct training/verification protocol."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the sampling procedure for BPR, dataset splits (train/test), prompt formatting, tokenization standardization, and the BadNets attack injection process."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Discussion and Limitations' provides substantive discussion of multiple specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 identifies specific threats: LoRA incompatibility, identical hardware requirement between trainer and auditor, and limitation to small models and fine-tuning only."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5: 'experiments focus on fine-tuning small LLMs, validation with larger models and alternative training procedures (pre-training, reinforcement learning) remains necessary to establish broader applicability.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model weights, training logs, distance measurements) is released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes data sources (Stanford Alpaca, AdvBench), sample sizes, train/test splits, and the BPR sampling procedure."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard public benchmarks (Stanford Alpaca, AdvBench)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents the pipeline: dataset selection → sampling with variable BPR → prompt formatting with alpaca template → tokenization → single-step training → verification comparison."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 acknowledges 'CEA List FactoryIA supercomputer, with financial support from the Île-de-France Regional Council.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors listed as Université Paris-Saclay, CEA LIST, Palaiseau, France. They are not evaluating their own commercial product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "CEA (French atomic energy commission) and Île-de-France Regional Council are public research funders with no financial stake in backdoor detection outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests a verification/defense protocol, not a pre-trained model's capability on knowledge benchmarks. Contamination is not relevant."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper evaluates a defense method, not model knowledge on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — contamination concepts do not apply to this defense evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 compares one-step training vs. verification time across all LLMs. The paper states verification is '3× faster than a training step' and up to 70% cost reduction."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is stated (single NVIDIA H100 GPU) but total GPU hours or compute budget for all experiments is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports results across multiple runs with uncertainty ranges, indicating seed/run variability is captured."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'multiple runs' (Section 4.1) but never states the exact number of runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No mention of hyperparameter search or how the chosen settings (lr=5e-5, batch size, etc.) were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters appear fixed across experiments with no explanation of how they were chosen or whether alternatives were tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed, so no multiple comparison correction is applied despite comparing across 4 models × 2 attacks × 4+ BPR levels."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgment of author-evaluation bias. The authors evaluate their own proposed protocol without independent evaluation or discussion of this bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 directly compares verification cost (time) against training cost, and Figure 3 shows detection performance as a function of layers verified (more layers = more compute)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether ASR on BadNets-style attacks adequately represents real-world backdoor threats, or whether the chosen benchmarks (Alpaca, AdvBench) capture realistic attack scenarios."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in the evaluation — the method directly compares weight tensors."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates a defense protocol, not model knowledge on benchmarks. Temporal leakage is not applicable — the models are fine-tuned in a controlled experiment."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation measures weight divergence, not model predictions on knowledge benchmarks. Feature leakage concepts do not apply."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation is a controlled experiment with known poisoned/clean splits. Non-independence in the benchmark sense is not applicable."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper tests a defense method, not a model's benchmark performance where leakage could inflate scores."
       }
     }
   },
   "claims": [
     {
       "claim": "Backdoor attacks can achieve high ASR with as little as 10% batch poisoning rate for Targeted Refusal attacks.",
       "evidence": "Table 1: Llama3.2-1B achieves 56.4% ASRtrigger at 10% BPR for Targeted Refusal. Most models reach 69-88% ASR at 25% BPR.",
       "supported": "strong"
     },
     {
       "claim": "PoTS detects backdoor attacks by analyzing only the LM-Head layer, with verification 3x faster than a training step.",
       "evidence": "Figure 3 shows ratio shifts at 10% BPR for most models (LM-Head only). Figure 4 shows verification time is up to 3x faster than training, particularly for Falcon 3-1B.",
       "supported": "moderate"
     },
     {
       "claim": "Adding posterior layers (LMH+1L, +3L, +5L) improves detection sensitivity, especially for models where LM-Head-only detection is weak.",
       "evidence": "Figure 3: Llama 3.2-1B shows minimal LM-Head-only sensitivity but clear detection with additional layers. Adding 3.7% of parameters (Falcon) significantly improves detection.",
       "supported": "strong"
     },
     {
       "claim": "Hidden malicious steps are more detectable than single-step poisoning, with attacker distances exceeding single-step poisoning by >3x with 3 hidden steps.",
       "evidence": "Figure 5 shows Qwen 0.5B LM-Head sensitivity increases substantially with hidden steps across all BPR values.",
       "supported": "moderate"
     },
     {
       "claim": "Jailbreaking attacks require substantially higher poisoning rates than Targeted Refusal to achieve comparable effectiveness.",
       "evidence": "Table 1: Jailbreaking ASR remains low (<20%) at 10-25% BPR for most models, while Targeted Refusal reaches 56-88% ASR at 10-25% BPR.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Title overclaims scope",
       "detail": "Paper title references 'Large Language Models' but experiments use only 0.5B-1.5B parameter models, which are small by current standards. The limitations section acknowledges this but the title does not."
     },
     {
       "flag": "No comparison with competing detection methods",
       "detail": "The paper positions against Proof-of-Learning and other verification methods conceptually but provides no direct numerical comparison. The only baseline is the clean/honest case."
     },
     {
       "flag": "Single attack method tested",
       "detail": "Only BadNets-style attacks are evaluated. More sophisticated backdoor methods (e.g., clean-label attacks, syntactic triggers) are not tested, limiting the generalizability of detection claims."
     },
     {
       "flag": "Number of runs not stated",
       "detail": "Table 1 reports uncertainty intervals from 'multiple runs' but never specifies how many runs were conducted, making it impossible to assess statistical power."
     },
     {
       "flag": "Identical hardware assumption",
       "detail": "The method requires identical hardware between trainer and auditor (acknowledged in limitations), which severely limits practical deployment scenarios."
     }
   ],
   "cited_papers": [
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison"],
       "year": 2024,
       "relevance": "Demonstrates that backdoors persist through safety alignment in LLMs, motivating verification-time detection approaches."
     },
     {
       "title": "BackdoorLLM: A comprehensive benchmark for backdoor attacks on large language models",
       "authors": ["Yige Li", "Hanxun Huang", "Yunhan Zhao", "Xingjun Ma", "Jun Sun"],
       "year": 2024,
       "arxiv_id": "2408.12798",
       "relevance": "Provides the backdoor benchmark and ASR evaluation methodology used in this paper's experiments."
     },
     {
       "title": "Proof-of-learning: Definitions and practice",
       "authors": ["Hengrui Jia", "Mohammad Yaghini"],
       "year": 2021,
       "relevance": "Foundational work on model training verification that PoTS extends and improves upon for LLM contexts."
     },
     {
       "title": "Tools for verifying neural models' training data",
       "authors": ["Dami Choi", "Yonadav Shavit", "David K Duvenaud"],
       "year": 2023,
       "relevance": "Prior verification approach for detecting unauthorized training data, which PoTS addresses for backdoor-specific scenarios."
     },
     {
       "title": "What does it take to catch a chinchilla? Verifying rules on large-scale neural network training via compute monitoring",
       "authors": ["Yonadav Shavit"],
       "year": 2023,
       "arxiv_id": "2303.11341",
       "relevance": "Addresses verifiability challenges in LLM training environments, directly related to PoTS's verification goals."
     },
     {
       "title": "TrustLLM: Trustworthiness in large language models",
       "authors": ["Lichao Sun", "Yue Huang"],
       "year": 2024,
       "relevance": "Comprehensive framework for LLM trustworthiness that contextualizes backdoor attacks as a safety concern."
     },
     {
       "title": "BadNets: Identifying vulnerabilities in the machine learning model supply chain",
       "authors": ["Tianyu Gu", "Brendan Dolan-Gavitt", "Siddharth Garg"],
       "year": 2017,
       "arxiv_id": "1708.06733",
       "relevance": "Foundational backdoor attack method used as the attack technique in PoTS experiments."
     },
     {
       "title": "Persistent pre-training poisoning of LLMs",
       "authors": ["Yiming Zhang", "Javier Rando"],
       "year": 2024,
       "arxiv_id": "2410.13722",
       "relevance": "Demonstrates persistence of backdoors through LLM training processes, supporting the need for training-time verification."
     },
     {
       "title": "Learning to poison large language models during instruction tuning",
       "authors": ["Yao Qiang", "Xiangyu Zhou"],
       "year": 2024,
       "arxiv_id": "2402.13459",
       "relevance": "Shows how crafted backdoor triggers achieve high ASR during instruction tuning, directly relevant to the attack model PoTS defends against."
     }
   ]
 }