ai-research-survey

scan.json (25437B)

---

 {
   "paper": {
     "title": "One Token Embedding Is Enough to Deadlock Your Large Reasoning Model",
     "authors": [
       "Mohan Zhang",
       "Yihua Zhang",
       "Jinghan Jia",
       "Zhangyang Wang",
       "Sijia Liu",
       "Tianlong Chen"
     ],
     "year": 2025,
     "venue": "NeurIPS 2025",
     "arxiv_id": "2510.15965",
     "doi": "10.48550/arXiv.2510.15965"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The Deadlock Attack achieves 100% attack success rate across four LRMs (Phi-RM, Nemotron-Nano, R1-Qwen, R1-Llama) and three math reasoning benchmarks by optimizing a single adversarial token embedding that induces perpetual reasoning loops. A key finding is the continuous-to-discrete projection gap: naïve projections of adversarial embeddings to token sequences nullify the attack, necessitating a backdoor implantation strategy. The attack is stealthy (negligible utility loss on benign inputs) and robust against existing overthinking mitigation strategies (CoD, CCoT, NoThinking).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository URL is provided in the abstract: https://github.com/UNITES-Lab/Deadlock-Attack."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks (GSM8K, MATH500, MMLU-Pro, AIME 2024, HumanEval, CommonsenseQA) which are all publicly accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency details are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. Training details are given (optimizer, learning rate, steps) but not packaged as reproduction instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 2-4 report only point estimates (ASR, Ave.Tokens, Ave.Time) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims of effectiveness are based on comparing raw numbers (e.g., 100% ASR vs baseline 2% ASR)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are implicitly clear from the results: baseline ASR of 0-6.98% vs 100% under attack, baseline ~500-1600 tokens vs 4000 tokens under attack. The magnitude of difference is reported with baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Main attack evaluations use 50 samples for GSM8K and MMLU-Pro, 43 samples for MATH500 Level 1, with no justification for these sample sizes. The 500-sample extended evaluation in Appendix C is larger but also unjustified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. Results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Baseline models without the attack are compared against attacked variants in all tables (Tables 2, 3, 4). Three mitigation strategies (CoD, CCoT, NoThinking) serve as additional baselines in Table 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The mitigation baselines (CoD, CCoT, NoThinking) are from 2025, and the attack models (R1-Qwen, R1-Llama, Nemotron-Nano, Phi-RM) are all recent 2025 models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 shows ablation studies on adversarial embedding length (L=1,2,5,10) and training set size (N=1,5,10,20). Additional ablations on learning rates in Appendix (Fig. A2)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: Attack Success Rate (ASR), Average Tokens (Ave.Tokens), and Average Time (Ave.Time). Stealthiness is evaluated via accuracy on benign inputs."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to this adversarial attack paper — the claims are about automated metrics (ASR, token counts, timing)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training uses 20 samples from MATH500 level 5 with 10 held for validation. Evaluation is on separate benchmarks (GSM8K, MMLU-Pro, full MATH500), distinct from the training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (4 models) and per benchmark (3 benchmarks) in Table 2, and per mitigation strategy in Table 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The continuous-to-discrete projection gap is extensively discussed as a failure mode (Section 3.3). Failed approaches (Gaussian smoothing, iterative projection) are analyzed in detail with Figures 2-4."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results are reported: naïve projection fails (Section 3.3), Gaussian smoothing fails (Fig. 3), iterative projection fails (Fig. 4), alternative distance metrics and PCA all fail (Appendix B)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 100% ASR across four LRMs and three benchmarks, which is supported by Table 2. Stealthiness and robustness claims are supported by Tables 3 and 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('the optimized embedding encourages transitional tokens') supported by controlled experiments: same model with/without the adversarial embedding, ablation studies isolating components."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Deadlock Your Large Reasoning Model' broadly, but only 4 models (all 3.8-8B parameter, all distilled/small reasoning models) are tested. No testing on larger models (70B+) or proprietary models (o1, DeepSeek-R1 full). The paper discusses future work on black-box attacks but doesn't bound current claims to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses the alternative explanation that models might naturally hit token limits on hard problems by testing on AIME (Appendix C) and showing baseline exhaustion rates of only 13-17% vs 93-100% under attack."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly defines its proxy: ASR is 'the percentage of instances where the model's generation reaches the maximum token limit' and notes the 4000-token limit is 'an experimental constraint for evaluation efficiency, not a limitation of the attack itself.' The extended 20,000-token evaluation validates this."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are given: Phi-4-Mini-Reasoning (3.8B), Llama-Nemotron-Nano (8B), DeepSeek-R1-Distill-Qwen-7B, DeepSeek-R1-Distill-Llama-8B. These are specific enough to identify exact models."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The attack operates at the embedding level, not through prompting. The mitigation strategies (CoD, CCoT) use prompts but these are from cited works, not novel to this paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A specifies: Adam optimizer, learning rate 1e-3, weight decay 0, β1=0.9, β2=0.999, 1000 training steps, L=1, max generation 4000 tokens. Training dataset details are also provided."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The attack modifies model embeddings directly."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes: 'selecting the first 30 samples from the MATH500 dataset at level 5. For each sample, we generated 100 distinct reasoning answers using the R1-Qwen model. Twenty of these samples formed the training set, while the remaining 10 served as a validation set.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion mentions future work directions but does not substantively discuss limitations of the current approach."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential confounds or weaknesses of the evaluation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what settings the results do NOT apply to (e.g., closed-source models, larger models, non-reasoning tasks). Future work mentions black-box extensions but doesn't frame current scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, per-example results) are released. Only aggregate statistics are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The training data collection is described: samples from MATH500 level 5, with 100 reasoning answers generated per sample using R1-Qwen. Evaluation benchmarks and sample counts are specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: select 30 MATH500 level-5 samples → generate 100 answers each with R1-Qwen → split 20 train / 10 validation → optimize adversarial embedding → implant as backdoor → evaluate on separate benchmarks."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgment section lists NSF grants (IIS-2207052, CNS-2235231, IIS-2338068), ARO Award, Cisco Research Award, Amazon Research Award, and IBM PhD Fellowship."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: UNC Chapel Hill, Michigan State University, UT Austin. No evaluated products are affiliated with these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders (NSF, ARO, Cisco, Amazon, IBM) are not directly invested in the outcome of this adversarial attack research. The models attacked (DeepSeek, Microsoft, NVIDIA) are not funders."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff dates for the evaluated models (R1-Qwen, R1-Llama, Phi-RM, Nemotron-Nano) are not stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the models may have seen GSM8K, MATH500, or MMLU-Pro during pre-training. This is relevant since models could already know the answers, affecting baseline behavior."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021), MATH500 (2023), and MMLU-Pro are public benchmarks that likely predate the training of these 2025 models. No contamination discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Average inference time per instance is reported in Table 2 (e.g., 118.41s for Phi-RM under attack vs 25.31s baseline). Training uses only 1000 steps with a single embedding vector."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget (GPU hours, hardware used, total training time) is stated. Only per-instance inference time is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. The LMC analysis (Fig. 2) uses two independently trained embeddings with different seeds but does not report seed sensitivity of the attack itself."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs for the main results is not stated. Results appear to be single-run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Learning rate ablation is shown (Fig. A2) but no systematic hyperparameter search budget is reported. It is unclear how the final hyperparameters were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The choice of L=1 and N=20 is justified through ablation studies (Fig. 5) showing L=1 is sufficient and N=20 provides robust generalization."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own attack method without acknowledging potential bias in their implementation or evaluation. No independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "The attack uses negligible compute (training a single embedding vector for 1000 steps). Compute budget differences are not a meaningful confound."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses construct validity by testing on AIME (Appendix C) to rule out false positives where models naturally hit token limits on hard problems, showing the attack effect is distinct from natural overthinking."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The attack modifies model embeddings directly."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GSM8K, MATH500, or MMLU-Pro solutions existed in the models' training data. The attack effectiveness could be influenced by whether models already know the answers."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The evaluation provides full problem text which could contain information the model has seen during training."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The adversarial embedding is trained on MATH500 level 5 samples and some evaluations are on MATH500. No discussion of whether this creates dependence between training and test data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Deadlock Attack achieves 100% attack success rate across four LRMs and three math reasoning benchmarks, forcing models to generate up to maximum token limits.",
       "evidence": "Table 2 shows 100% ASR for all 12 model-benchmark combinations with 4000-token limit. Table A1 shows 93-100% ASR with 20,000-token limit.",
       "supported": "strong"
     },
     {
       "claim": "The attack is stealthy: backdoored models maintain near-identical performance on benign inputs without the trigger.",
       "evidence": "Table 4 shows accuracy differences within ±4pp on 50-sample evaluations. Table A2 shows negligible differences on 500-sample evaluations across 6 benchmarks.",
       "supported": "moderate"
     },
     {
       "claim": "Existing overthinking mitigation strategies (CoD, CCoT, NoThinking) fail to defend against the attack.",
       "evidence": "Table 3 shows 100% ASR and high Ave.Tokens across all mitigation strategies and models on GSM8K.",
       "supported": "strong"
     },
     {
       "claim": "There is a substantial continuous-to-discrete projection gap that prevents naïve conversion of adversarial embeddings to discrete tokens.",
       "evidence": "Figure 2 shows LMC analysis where projected embeddings consistently fail. Figures 3-4 show Gaussian smoothing and iterative projection also fail. Figure A1 shows failures across multiple distance metrics.",
       "supported": "strong"
     },
     {
       "claim": "A single adversarial token embedding (L=1) is sufficient for the attack.",
       "evidence": "Figure 5(L) shows L=1 converges, and main experiments (Tables 2-4) use L=1 with 100% ASR.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Small evaluation sample sizes",
       "detail": "Main attack and stealthiness evaluations use only 50 samples per benchmark (43 for MATH500 L1). While the 500-sample extended evaluation in Appendix C is more robust, the main results could be affected by sample selection. No uncertainty quantification is provided."
     },
     {
       "flag": "Only small models tested",
       "detail": "All four tested models are 3.8-8B parameters — small, distilled reasoning models. The attack's effectiveness on larger models (70B+) or proprietary reasoning models (o1, full DeepSeek-R1) is unknown, yet the title and claims are broadly stated."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations section. The conclusion mentions future directions but doesn't honestly discuss what the current work does not show."
     },
     {
       "flag": "Single-run results without variance",
       "detail": "All experimental results appear to be from single runs with no variance reporting, error bars, or confidence intervals, despite the stochastic nature of the training process and model generation."
     }
   ],
   "cited_papers": [
     {
       "title": "Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Source of two of the four victim models (R1-Qwen and R1-Llama distillations) and a key LRM architecture."
     },
     {
       "title": "Overthink: Slowdown attacks on reasoning LLMs",
       "authors": ["A. Kumar", "J. Roh", "A. Naseh", "M. Karpinska", "M. Iyyer", "A. Houmansadr", "E. Bagdasarian"],
       "year": 2025,
       "arxiv_id": "2502.02542",
       "relevance": "Most closely related prior work on resource exhaustion attacks against LRMs, using decoy problem injection rather than embedding manipulation."
     },
     {
       "title": "Sleeper agents: Training deceptive llms that persist through safety training",
       "authors": ["E. Hubinger", "C. Denison", "J. Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Related work on backdoor persistence in LLMs through safety training — relevant to AI safety and supply chain security."
     },
     {
       "title": "Do not think that much for 2+3=? On the overthinking of o1-like LLMs",
       "authors": ["X. Chen", "J. Xu", "T. Liang"],
       "year": 2024,
       "relevance": "Analyzes the overthinking phenomenon in reasoning models that the Deadlock Attack exploits as a vulnerability surface."
     },
     {
       "title": "Adversarial ml problems are getting harder to solve and to evaluate",
       "authors": ["J. Rando", "J. Zhang", "N. Carlini", "F. Tramèr"],
       "year": 2025,
       "arxiv_id": "2502.02260",
       "relevance": "Discusses the continuous-to-discrete gap in adversarial attacks on LLMs that this paper addresses with its backdoor mechanism."
     },
     {
       "title": "Badchain: Backdoor chain-of-thought prompting for large language models",
       "authors": ["Z. Xiang", "F. Jiang", "Z. Xiong"],
       "year": 2024,
       "arxiv_id": "2401.12242",
       "relevance": "Related backdoor attack on CoT reasoning in LLMs targeting reasoning accuracy rather than resource exhaustion."
     },
     {
       "title": "Scaling llm test-time compute optimally can be more effective than scaling model parameters",
       "authors": ["C. Snell", "J. Lee", "K. Xu", "A. Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Foundational work on test-time compute scaling in LLMs that motivates the resource exhaustion attack surface."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark used for stealthiness evaluation of the backdoored models on code generation."
     },
     {
       "title": "Cats confuse reasoning llm: Query agnostic adversarial triggers for reasoning models",
       "authors": ["M. Rajeev", "R. Ramamurthy", "P. Trivedi"],
       "year": 2025,
       "arxiv_id": "2503.01781",
       "relevance": "Related work on query-agnostic adversarial triggers for reasoning models, targeting accuracy rather than resource exhaustion."
     },
     {
       "title": "ShadowCoT: Cognitive hijacking for stealthy reasoning backdoors in LLMs",
       "authors": ["G. Zhao", "H. Wu", "X. Zhang"],
       "year": 2025,
       "arxiv_id": "2504.05605",
       "relevance": "Related backdoor attack on LLM reasoning processes — manipulates reasoning paths for stealthy behavior."
     }
   ]
 }