ai-research-survey

scan.json (24959B)

---

 {
   "paper": {
     "title": "Backdoor Samples Detection Based on Perturbation Discrepancy Consistency in Pre-trained Language Models",
     "authors": [
       "Zuquan Peng",
       "Jianming Fu",
       "Lixin Zou",
       "Li Zheng",
       "Yanzhen Ren",
       "Guojun Peng"
     ],
     "year": 2025,
     "venue": "Neural Networks (preprint)",
     "arxiv_id": "2509.05318"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository URL is provided in Section 5 (footnote 1): https://github.com/pzq7025/BackdoorDetection."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: YELP, OLID, and COVID, and uses public toolkits OpenBackdoor and TextAttack for attack generation. All datasets are referenced with citations and are publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 specifies: Ubuntu OS v20.04, Intel Xeon E5-2680 CPU, 126GB RAM, Quadro RTX 5000 GPU, Python v3.7.5, PyTorch v1.13.0+cu117, OpenAttack v2.1.1, OpenBackdoor v1.0.0, TextAttack v0.3.7, and Transformers library v3.3.0."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a code repository link is provided and the environment is specified, the paper does not include step-by-step reproduction instructions, a README with commands, or a dedicated 'Reproducing Results' section. The experimental setup describes parameters but not how to run the experiments end-to-end."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (Tables 4, 5, 7, 8 and figures) report single AUROC point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims NETE outperforms existing methods based solely on comparing AUROC values in tables. No statistical significance tests (p-values, t-tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw AUROC values but does not provide effect sizes (e.g., Cohen's d) or contextualize improvements with baseline context beyond raw numbers. The absolute AUROC differences are visible but not characterized as effect sizes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 500 samples for the PDC test (Sec. 4.2) and 10,000 samples (5,000 clean + 5,000 backdoor) for runtime evaluation (Sec. 6.3), but no justification or power analysis is provided for why these specific sample sizes were chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across runs, or multiple-run results with spread measures are reported anywhere in the paper. All results appear to be single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against five zero-shot detection baselines: Log, Rank, LogRank, Entropy, and ONION (Section 6.1). Additionally, roberta-base and roberta-large are compared for potential attack detection (Sec. 6.4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include ONION (2021), STRIP (2019), and the paper discusses recent works like Len-Meo (2024) and UnToken (2025). The paper explicitly justifies why white-box methods are excluded (they require poisoned model access) and discusses recent defense literature in Section 2.2."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 includes ablation-style analyses: impact of perturbation numbers (Fig. 4), impact of different pre-trained scoring models (GPT-xl vs GPT-medium, Fig. 5), and impact of different mask-filling models (T5-small vs T5-base vs T5-large, Fig. 6)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only AUROC is used as the evaluation metric throughout the paper. Section 6.1 justifies the choice of AUROC but no additional metrics (e.g., precision, recall, F1) are reported."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a detection method evaluated entirely via automated metrics (AUROC) on programmatically constructed backdoor/clean samples. Human evaluation of the system's outputs is not relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not clearly describe a separation between development/tuning sets and test sets. The threshold analysis in Sec. 6.3 uses word-level backdoor samples to set thresholds for other attacks, but there is no explicit held-out test set protocol."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by attack type (word, sentence, syntactic, style with 4 sub-styles), by dataset (COVID, OLID, YELP), and by additional complex scenarios in Table 7 (CBA, BadChain, BadEdit, VPI, Sleepagent across 13 datasets)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 explicitly acknowledges that 'current black-box detection methods, including our proposed approach, show relatively poor detection performance under word and sentence insertion' and explains why. Section 7.2 discusses limitations in correcting syntactic/style backdoor samples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that word-level and sentence-level attacks are harder to detect (Tables 4-5 show lower AUROC for these), and Section 6.2 explicitly discusses the suboptimal performance on these attack types. The threshold analysis in Sec. 6.3 shows only marginal improvement over ONION (0.77 vs 0.76)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims NETE 'outperforms existing zero-shot black-box detection methods,' which is supported by Tables 4, 5, 7, and 8 showing NETE achieves the highest AUROC in most settings. The abstract also claims applicability to both pre-training and post-training phases, which is discussed in Sections 3 and 6.4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims center on the perturbation discrepancy consistency phenomenon. The claim that backdoor samples exhibit smaller perturbation discrepancy is supported by both empirical observation (Fig. 1, Sec. 4.2) and mathematical derivation connecting perturbation discrepancy to curvature (Sec. 5.3). Ablation studies in Sec. 6.3 provide controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper tests across multiple attack types (4 traditional + 5 LLM backdoor attacks + adversarial + jailbreak), multiple datasets (3 core + 13 additional), and acknowledges limitations for word/sentence-level attacks. Section 7.3 explicitly states time consumption limitations for large-scale datasets."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why NETE works beyond the perturbation discrepancy phenomenon. No threats-to-validity or confound analysis is provided. The paper does not consider whether the performance gains could be attributed to other factors."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-2', 'GPT-medium', 'GPT-xl', 'T5-small', 'T5-base', 'T5-large' without specific version identifiers or snapshot dates. For the LLM-adversary experiment, 'ChatGPT (i.e., GPT-3.5)' is used without a specific API version."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The method does not use prompting. It computes log probabilities and generates perturbations via mask-filling, which are algorithmic operations not involving prompt design."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: mask-filling ratio of 10%, span of up to 2 words (Sec. 5.2), 50/200 perturbation numbers (Sec. 6.3), batch size of 32 (Sec. 6.3), cosine similarity 0.8 for adversarial attacks, 3 injected triggers for word-level attacker, temperature settings {0.3, 0.5, 0.7} for Temperature-adversary."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method is a standalone detection algorithm without agent-like components."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 describes the attack generation process using OpenBackdoor with specified parameters, the STRAP model for style transfer, TextAttack for adversarial examples, and specific settings (cosine similarity 0.8, max 3 perturbed words). Dataset statistics are provided in Table 3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 is titled 'Limitation and Future Work' and provides substantive discussion of time consumption issues and strategies for improvement."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 identifies a specific limitation: increasing data length leads to greater time expenditure, which 'imposes a substantial adverse effect on real-time detection systems.' Section 7.1 discusses the specific design choice of low span setting and its implications. Section 7.2 discusses the specific limitation that correction is only possible for word/sentence-level triggers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper mentions some limitations, it does not explicitly state what the results do NOT show or what populations/settings are excluded. The paper does not systematically bound its claims to specific domains or model types beyond the tested ones."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not release raw experimental data (e.g., per-sample detection scores, generated perturbation samples, or detailed per-experiment logs). Only aggregated AUROC values are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 describes how datasets were selected (YELP, OLID, COVID from real-world scenarios), how backdoor samples were generated (using OpenBackdoor with specified parameters), how adversarial examples were created (TextAttack with cosine similarity 0.8), and Table 3 provides dataset statistics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Data sources are standard public benchmark datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: (1) clean datasets selected (Table 3), (2) backdoor samples generated via OpenBackdoor with four trigger schemes (Sec. 6.1), (3) perturbations generated via T5 mask-filling (Sec. 5.2), (4) log probabilities computed via GPT models (Sec. 5.2), (5) curvature threshold applied (Sec. 5.2, Algorithm 1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledge section states: 'This research was supported by the National Natural Science Foundation of China (Grant No. 62272351).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Wuhan University, Key Laboratory of Aerospace Information Security and Trusted Computing, Ministry of Education. No commercial product is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is the National Natural Science Foundation of China, a government research funding agency with no financial stake in the specific detection method's performance."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. It uses pre-trained models (GPT-2, T5) as tools for computing log probabilities and generating perturbations, not to test their knowledge. The evaluation is of the detection method, not of model knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale: the paper evaluates a detection algorithm, not a pre-trained model's performance on a knowledge benchmark. Train/test overlap for the language models is not relevant to the detection task."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale: benchmark contamination is not relevant since the paper tests a detection method, not model knowledge or generative capability on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 reports GPU memory consumption (4166 MB for models) and detailed runtime results in Table 6 across datasets and trigger types, with per-sample timing data."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 provides comprehensive runtime and GPU memory consumption data. The hardware is specified (Quadro RTX 5000). Runtime ranges from ~35 seconds to ~650 seconds per 10,000 samples depending on dataset and trigger type."
       }
     }
   },
   "claims": [
     {
       "claim": "NETE outperforms existing zero-shot black-box detection methods for backdoor sample detection across multiple attack types and datasets.",
       "evidence": "Tables 4, 5, 7, and 8 show NETE achieves the highest AUROC in the majority of experimental settings across style, word, sentence, and syntactic attacks on COVID, OLID, and YELP datasets, as well as 5 LLM-specific backdoor attacks across 13 datasets.",
       "supported": "strong"
     },
     {
       "claim": "Backdoor samples exhibit anomalous perturbation discrepancy consistency, with smaller perturbation discrepancy compared to clean samples.",
       "evidence": "Figure 1 and Section 4.2 demonstrate the density distribution gap between backdoor and clean samples across four trigger types on the YELP dataset using 500 samples. Section 5.3 provides mathematical justification connecting this to curvature.",
       "supported": "strong"
     },
     {
       "claim": "The detection performance converges at approximately 50 perturbation steps.",
       "evidence": "Figure 4 (Section 6.3) shows AUROC plateaus around 50 perturbations across all four attack types and three datasets.",
       "supported": "moderate"
     },
     {
       "claim": "The choice of pre-trained scoring model and mask-filling model has minimal impact on detection results.",
       "evidence": "Figures 5 and 6 compare GPT-xl vs GPT-medium and T5-small vs T5-large respectively, showing negligible differences. However, these comparisons are limited to models within the same family.",
       "supported": "moderate"
     },
     {
       "claim": "NETE can effectively detect adversarial examples and jailbreak samples.",
       "evidence": "Figure 8 shows NETE outperforms ONION on Textbugger and Textfooler adversarial examples. Section 6.4 reports AUROC of 0.91 for jailbreak detection vs 0.74 for Log baseline. However, only one jailbreak attack type (linguistic mutations) is tested.",
       "supported": "moderate"
     },
     {
       "claim": "Current black-box methods, including NETE, show relatively poor detection for word-level and sentence-level attacks.",
       "evidence": "Table 5 shows lower AUROC values for word and sentence attacks compared to syntactic and style attacks. Section 6.2 provides an explanation based on rare words occurring within normal output distributions.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper proposes NETE, a zero-shot black-box backdoor sample detection method based on the observation that backdoor samples exhibit smaller perturbation discrepancy than clean samples. The method uses T5 mask-filling to generate perturbations and GPT-2 to compute log probabilities, then applies a curvature-based threshold to classify samples. NETE outperforms existing zero-shot detection methods (Log, Rank, LogRank, Entropy, ONION) across four traditional and five LLM-specific backdoor attacks on multiple datasets, while also showing effectiveness against adversarial examples and jailbreak attacks. The method is lightweight, requiring only 4166 MB GPU memory, and detection performance converges at ~50 perturbations.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All AUROC results are single point estimates without confidence intervals, error bars, or variance across runs. Without knowing the variability, it is impossible to determine whether performance differences between methods are meaningful or within noise."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims of outperforming baselines are based solely on comparing raw AUROC values. Some differences are very small (e.g., 0.77 vs 0.76 in threshold analysis) and could be within random variation."
     },
     {
       "flag": "Single evaluation metric",
       "detail": "Only AUROC is reported. In practical deployment, precision and recall at specific thresholds matter (e.g., high false positive rates could make the system unusable). The paper does not report precision-recall curves or F1 scores."
     },
     {
       "flag": "Threshold selection relies on known attack knowledge",
       "detail": "The threshold analysis in Section 6.3 uses word-level backdoor samples (known attacks) to set thresholds for unknown attacks, which contradicts the zero-shot claim to some extent. The practical question of how to set thresholds without any backdoor samples is not fully addressed."
     }
   ],
   "cited_papers": [
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["E. Hubinger", "C. Denison", "J. Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Directly relevant to AI safety: demonstrates persistent deceptive behaviors in LLMs that survive safety training, used as an attack scenario in this paper."
     },
     {
       "title": "BackdoorLLM: A comprehensive benchmark for backdoor attacks on large language models",
       "authors": ["Y. Li", "H. Huang", "Y. Zhao"],
       "year": 2025,
       "arxiv_id": "2408.12798",
       "relevance": "Benchmark framework for evaluating backdoor attacks on LLMs, used to ensure consistency in this paper's empirical analysis."
     },
     {
       "title": "BadChain: Backdoor chain-of-thought prompting for large language models",
       "authors": ["Z. Xiang", "F. Jiang", "Z. Xiong"],
       "year": 2024,
       "relevance": "Novel backdoor attack exploiting chain-of-thought prompting in LLMs, relevant to agentic AI security."
     },
     {
       "title": "BadEdit: Backdooring large language models by model editing",
       "authors": ["Y. Li", "T. Li", "K. Chen"],
       "year": 2024,
       "relevance": "Backdoor injection through model editing rather than poisoning, relevant to LLM security research."
     },
     {
       "title": "Composite backdoor attacks against large language models",
       "authors": ["H. Huang", "Z. Zhao", "M. Backes"],
       "year": 2024,
       "relevance": "Multi-component backdoor attacks targeting LLMs with composite triggers, relevant to understanding attack complexity."
     },
     {
       "title": "Backdooring instruction-tuned large language models with virtual prompt injection",
       "authors": ["J. Yan", "V. Yadav", "S. Li"],
       "year": 2024,
       "relevance": "Backdoor attacks targeting instruction-tuned LLMs, relevant to safety of agentic LLM systems."
     },
     {
       "title": "Poison attack and poison detection on deep source code processing models",
       "authors": ["J. Li", "Z. Li", "H. Zhang"],
       "year": 2024,
       "doi": "10.1145/3605943",
       "relevance": "Backdoor attacks and detection specifically on code processing models, directly relevant to LLM programming safety."
     },
     {
       "title": "DetectGPT: Zero-shot machine-generated text detection using probability curvature",
       "authors": ["E. Mitchell", "Y. Lee", "A. Khazatsky"],
       "year": 2023,
       "relevance": "Foundational method for zero-shot text detection using curvature that NETE builds upon, relevant to LLM-generated content detection."
     },
     {
       "title": "BDMMT: Backdoor sample detection for language models through model mutation testing",
       "authors": ["J. Wei", "M. Fan", "W. Jiao"],
       "year": 2024,
       "relevance": "Recent backdoor detection method using model mutation testing, relevant baseline for LLM security."
     },
     {
       "title": "ArtPrompt: ASCII art-based jailbreak attacks against aligned LLMs",
       "authors": ["F. Jiang", "Z. Xu", "L. Niu"],
       "year": 2024,
       "relevance": "Novel jailbreak attack method against aligned LLMs, relevant to AI safety evaluation."
     },
     {
       "title": "CodeT5+: Open code large language models for code understanding and generation",
       "authors": ["Y. Wang", "H. Le", "A. Gotmare"],
       "year": 2023,
       "relevance": "Code LLM that could be targeted by backdoor attacks, relevant to understanding the scope of backdoor threats in code generation."
     }
   ]
 }