ai-research-survey

scan.json (27138B)

---

 {
   "paper": {
     "title": "BI-GRPO: Bidirectional Optimization for Jailbreak Backdoor Injection on LLMs",
     "authors": [
       "Wence Ji",
       "Jiancan Wu",
       "Aiying Li",
       "Shuyi Zhang",
       "Junkang Wu",
       "An Zhang",
       "Xiang Wang",
       "Xiangnan He"
     ],
     "year": 2025,
     "venue": "Preprint (arXiv)",
     "arxiv_id": "2509.19775"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly states 'We do not release the trained backdoored models and collected data to the public, researchers wishing to reproduce or extend our work must contact the authors' (Appendix J). No repository URL or code archive is provided. This is 'available upon request' which counts as NO."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: Anthropic RLHF dataset (harmless-base subset), DAN, DNA, Addition, StrongREJECT, and ADVbench. All evaluation datasets are standard public benchmarks referenced with citations."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '4 NVIDIA A100 GPUs (80GB)' and the 'verl' framework with 'mixed-precision (fp16) training' (Appendix C.3), but does not provide a requirements.txt, Dockerfile, or detailed library version specifications. There is no mention of Python version, PyTorch version, or other dependency versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the paper describes the method, hyperparameters, and prompt templates, there are no step-by-step reproduction instructions, no README with commands, and no scripts to replicate experiments. Code is not released, making reproduction impossible."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '99.7', '100') without confidence intervals, error bars, or any uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims bi-GRPO 'significantly advancing the state-of-the-art' and 'overwhelmingly outperforms all baselines' but no statistical significance tests (p-values, t-tests, etc.) are used to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout. For example, Table 2 shows bi-GRPO achieves 99.7% ASR on DAN vs. Sleeper at 42.7% and Poison-RLHF at 66.1%, providing both absolute numbers and clear baseline comparison context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The training set uses 1,000 randomly sampled entries from the Anthropic RLHF dataset. The GPT-4 and human evaluations use 100 randomly selected queries. No justification is provided for why these specific sample sizes were chosen, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. Results appear to be single-run numbers with no indication of whether experiments were repeated or how stable the results are across random seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against three baselines: Sleeper (SFT-based), Poison-RLHF, and JailbreakEdit (model editing), covering the three main paradigms for jailbreak backdoor injection (Section 4.1, Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are from 2024-2025: Sleeper (Hubinger et al., 2024), Poison-RLHF (Rando & Tramer, 2024), and JailbreakEdit (Chen et al., 2025). These represent the current state of the art in jailbreak backdoor attacks."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 presents an ablation study removing the pairwise reward mechanism and the pairwise rollout strategy, demonstrating the contribution of each component to overall performance (Figure 5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple evaluation metrics: ASR via LLaMA-Guard-3, ASR via Longformer classifier, Combined Success Rate, GPT-4 win rate for malicious helpfulness, human evaluation win rate, valid ratio, and MMLU for general capability (Tables 1-3, 7, 10)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Five human experts independently evaluated 100 jailbreak responses, selecting the most useful harmful response among four attack methods. Inter-rater agreement is reported (47% full agreement, 99% majority agreement). Evaluators were paid 200 RMB each (Appendix I)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training uses 1,000 entries from the Anthropic RLHF harmless-base subset, while evaluation uses five separate benchmark datasets (DAN, DNA, Addition, StrongREJECT, ADVbench) that are explicitly distinct from the training data. Section 4.4.1 further tests on out-of-distribution harmful categories."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (Tables 1, 2), per model (Table 1), per harmful intent category (Figure 4, ten categories), and per trigger type (Table 8). The ablation study also provides per-step breakdowns (Figure 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss failure cases of bi-GRPO. All presented results show near-perfect attack success rates. No analysis of when or why bi-GRPO might fail is provided."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows bi-GRPO outperforming baselines. No negative results, failed configurations, or approaches that were tried and abandoned are reported. The slight increase in non-triggered ASR on the Addition dataset is not discussed as a concern."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims >99% ASR, preserved stealthiness, and superior malicious helpfulness. Table 1 confirms ASR near or at 100% with triggers, Table 2 confirms low ASR without triggers, and Table 3 confirms 75-79% win rate in malicious helpfulness evaluations."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (Section 4.5) that remove individual components (pairwise reward, pairwise rollout) and measure performance degradation. This controlled single-variable manipulation design adequately supports the causal claims about component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to 'significantly advance the state-of-the-art in jailbreak backdoor attacks' and states capabilities 'fully generalize to arbitrary unseen prompts.' However, experiments are limited to 7B-14B parameter models from two families (Llama2, Qwen2.5). The conclusion in Section 5 does note the limitation to open-source models, but the generalization claims in the abstract and introduction are broader than what was tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For instance, it does not consider whether the superior performance could be partly due to differences in training data amounts, hyperparameter tuning effort, or the specific choice of safety classifier (LLaMA-Guard) rather than the pairwise mechanism itself."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies Llama-2-7b-chat, Qwen2.5-7b-instruct, Qwen2.5-14b-instruct as target models, Llama-Guard-3-8b as the safety judge, and gpt-4-0613 for GPT-4 evaluation (Appendix I). The GPT-4 version is specifically noted as 'gpt-4-0613.'"
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides full prompt templates for both Qwen2.5-instruct and Llama2-chat in Figure 6 (Appendix C.1), and the full GPT-4 evaluation prompt in Figure 9 (Appendix I). The actual prompt structure with system instructions and question placeholders is given in full."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.2 reports: learning rate 1e-6, n=8 response pairs, batch size of 8 prompts, minibatch 32, microbatch 8, reward score range ±3, length reward coefficient 1/1024. Testing: temperature=0, top_p=1.0, max_tokens=1024. Sensitivity analysis is provided for key hyperparameters (Tables 5-6)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. It is an RL-based training method, not an agent system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the data preparation: 'we randomly sample 1,000 entries and retain only the first user query from each multi-turn dialogue as training instances' from the harmless-base subset of the Anthropic RLHF dataset. Appendix C.4 provides further details on how baseline training data was constructed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The conclusion (Section 5) includes a limitation statement: 'A limitation is that these attack paradigms are based on reinforcement learning, which require fine-tuning LLMs' parameters. This makes the method impractical for closed-source LLMs, where access to the model's internals is restricted.' Appendix J discusses broader impacts and safeguards."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The only limitation mentioned is that the method requires model parameter access (closed-source limitation). There is no discussion of specific threats to validity such as the reliance on LLaMA-Guard as a safety judge, potential biases in the evaluation setup, or whether the results depend on specific model architectures."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its claims to specific model sizes, architectures, or attack scenarios. The closed-source limitation is mentioned but other scope boundaries (e.g., not tested on models >14B, not tested on non-English content, not tested against diverse defense methods beyond BAIT) are not stated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We do not release the trained backdoored models and collected data to the public' (Appendix J). Raw experimental outputs and intermediate results are not available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes data collection: 1,000 entries randomly sampled from the harmless-base subset of the Anthropic RLHF dataset, retaining only the first user query. Evaluation datasets (DAN, DNA, Addition, StrongREJECT, ADVbench) are referenced with citations."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Five human experts are mentioned for human evaluation (Appendix I), but no details are provided about how these experts were recruited, what their domain expertise is, or whether the recruitment process could introduce bias. The paper only says 'five human experts independently vote' and that they were paid 200 RMB."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: training data from Anthropic RLHF → random sampling of 1,000 entries → first user query extraction → trigger augmentation. Evaluation pipeline: five benchmark datasets → model inference with/without trigger → safety classification by LLaMA-Guard/Longformer → ASR computation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. There is no mention of grants, corporate sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed with their affiliation: University of Science and Technology of China. This is an academic institution without an obvious commercial interest in the evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence of the funder cannot be assessed. The absence of funding disclosure means this criterion cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement or financial interests declaration in the paper. Absence of disclosure is not absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on a benchmark. It trains backdoors into models and evaluates the attack effectiveness. The training cutoff of the base models is not relevant to the claims about attack success."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper is about injecting backdoors, not evaluating model knowledge on benchmarks. Train/test overlap in the traditional contamination sense is not applicable. The training and evaluation datasets are explicitly separate (Anthropic RLHF for training, five distinct benchmarks for evaluation)."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Benchmark contamination is not applicable here as the paper evaluates attack success rates on harmful prompt datasets, not the model's knowledge or reasoning capability on traditional benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The paper includes a human evaluation study with five annotators. No pre-registration is mentioned."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation involves experts evaluating harmful content (jailbreak responses). No IRB or ethics board approval is mentioned despite the sensitive nature of the content being evaluated."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes evaluators only as 'five human experts' and 'five annotators.' No demographics are reported: no experience level, domain of expertise, gender, or geographic information."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for the human evaluators are stated. The paper does not describe what qualifies someone as a 'domain expert' in this context."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study with treatment/control groups for participants. The human evaluation is a judging task, not a randomized experiment."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the responses were 'made into an anonymous questionnaire' (Appendix I), and the GPT-4 evaluation uses shuffled model identifiers (A, B, C, D). The human evaluators appear to have been blinded to which method produced each response."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information about attrition is provided. The paper does not state whether all five evaluators completed all 100 evaluations or whether any dropped out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example cost is reported. The paper does not mention how long inference takes or the cost of generating responses from the backdoored models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using '4 NVIDIA A100 GPUs (80GB)' for 7B models and '8 A100 GPUs' for 14B models (Appendix C.3), but does not state training time, total GPU hours, or total computational cost."
       }
     }
   },
   "claims": [
     {
       "claim": "bi-GRPO achieves attack success rate exceeding 99% across all tested models and datasets when the trigger is present.",
       "evidence": "Table 1 shows ASR values of 98.8-100% across DAN, DNA, Addition, StrongREJECT, and ADVbench datasets on Llama2-7b, Qwen2.5-7b, and Qwen2.5-14b.",
       "supported": "strong"
     },
     {
       "claim": "bi-GRPO preserves the model's original safety alignment when no trigger is present (stealthiness).",
       "evidence": "Table 1 shows non-triggered ASR remains low (0.0-12.1%) for the attacked models, close to clean model baselines. MMLU evaluation in Appendix H shows negligible general capability degradation (e.g., Qwen2.5-7b: 74.1% clean vs 73.6% backdoored).",
       "supported": "strong"
     },
     {
       "claim": "bi-GRPO produces the most maliciously helpful jailbreak responses compared to baselines.",
       "evidence": "Table 3 shows bi-GRPO achieves 79% GPT-4 win rate and 75% human win rate, far exceeding JailbreakEdit (16%/22%), Sleeper (4%/3%), and Poison-RLHF (1%/0%).",
       "supported": "strong"
     },
     {
       "claim": "The injected backdoor generalizes across all types of harmful intent, including categories not seen during training.",
       "evidence": "Section 4.4.1 and Figure 4a show ASR >96% across all ten harmful categories when trained only on non-violent crime data.",
       "supported": "moderate"
     },
     {
       "claim": "bi-GRPO evades state-of-the-art backdoor detection (BAIT).",
       "evidence": "Appendix G reports that BAIT failed to detect the backdoor, outputting 'is_backdoor: false' with all detection metrics at 0.000. However, only one defense method was tested.",
       "supported": "weak"
     },
     {
       "claim": "Both pairwise rollout and pairwise reward mechanisms are necessary for bi-GRPO's performance.",
       "evidence": "Section 4.5 and Figure 5 show that removing pairwise reward reduces Combined Success Rate, and further removing pairwise rollout degrades performance even more.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "bi-GRPO is a reinforcement learning framework for injecting jailbreak backdoors into LLMs that achieves >99% attack success rate with triggers while preserving safety alignment without triggers. The method uses pairwise rollouts and rule-based pairwise rewards to jointly optimize for effectiveness, stealthiness, and malicious helpfulness. Experiments across three open-source model families (7B-14B parameters) and five harmful prompt datasets demonstrate bi-GRPO substantially outperforms SFT-based, model editing, and RLHF-based backdoor injection methods. The approach also evades the BAIT backdoor detection method, though only one defense was tested.",
   "red_flags": [
     {
       "flag": "No error bars or variance reporting",
       "detail": "All experimental results are reported as single point estimates without confidence intervals, standard deviation, or any indication of whether experiments were repeated. For a method involving RL training with stochastic sampling, results can vary significantly across random seeds."
     },
     {
       "flag": "Results appear too clean",
       "detail": "bi-GRPO achieves near-perfect ASR (99-100%) across all models and datasets with trigger, and simultaneously near-zero ASR without trigger. No failure cases or edge cases are discussed. The consistently perfect results across all conditions warrant scrutiny."
     },
     {
       "flag": "Only one defense method tested",
       "detail": "The paper tests against only BAIT for backdoor detection evasion and claims robustness against 'state-of-the-art detection methods.' Testing against a single defense provides very limited evidence of evasion capability."
     },
     {
       "flag": "Missing failure analysis",
       "detail": "The paper reports no failure cases, negative results, or configurations that did not work. Every experiment shows bi-GRPO as clearly superior, which raises questions about selective reporting."
     },
     {
       "flag": "Limited model scale evaluation",
       "detail": "Experiments are limited to 7B-14B models from only two families (Llama2, Qwen2.5). Generalization claims to 'safety-aligned LLMs' broadly are not supported by such a narrow evaluation scope."
     },
     {
       "flag": "No funding disclosure",
       "detail": "The paper contains no funding disclosure or acknowledgments section. While the authors are at an academic institution, the absence of any funding information is notable."
     }
   ],
   "cited_papers": [
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Foundational work on backdoor persistence through safety training, directly compared as a baseline in this paper."
     },
     {
       "title": "Universal jailbreak backdoors from poisoned human feedback",
       "authors": ["Javier Rando", "Florian Tramer"],
       "year": 2024,
       "relevance": "Key baseline (Poison-RLHF) for RLHF-based backdoor injection that bi-GRPO aims to improve upon."
     },
     {
       "title": "Injecting universal jailbreak backdoors into LLMs in minutes",
       "authors": ["Zhuowei Chen", "Qiannan Zhang", "Shichao Pei"],
       "year": 2025,
       "relevance": "JailbreakEdit baseline for model editing-based backdoor injection, representing an alternative paradigm to RL-based approaches."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Inspiration for using GRPO in LLM post-training; bi-GRPO adapts the GRPO framework for adversarial purposes."
     },
     {
       "title": "A StrongREJECT for empty jailbreaks",
       "authors": ["Alexandra Souly"],
       "year": 2024,
       "relevance": "Provides one of the key evaluation benchmarks (StrongREJECT) used for measuring jailbreak attack effectiveness."
     },
     {
       "title": "BAIT: Large language model backdoor scanning by inverting attack target",
       "authors": ["Guangyu Shen"],
       "year": 2025,
       "relevance": "State-of-the-art backdoor detection method that bi-GRPO claims to evade, relevant to evaluating defenses against backdoor attacks."
     },
     {
       "title": "Jailbreak attacks and defenses against large language models: A survey",
       "authors": ["Sibo Yi"],
       "year": 2024,
       "arxiv_id": "2407.04295",
       "relevance": "Comprehensive survey of jailbreak attack and defense landscape, providing context for backdoor attack research."
     },
     {
       "title": "The jailbreak tax: How useful are your jailbreak outputs?",
       "authors": ["Kristina Nikolic"],
       "year": 2025,
       "relevance": "Evaluates the practical usability of jailbreak outputs, directly relevant to the malicious helpfulness metric used in this paper."
     },
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Yuntao Bai"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "Source of the Anthropic RLHF dataset used for training in this paper, foundational work on RLHF alignment."
     },
     {
       "title": "BackdoorLLM: A comprehensive benchmark for backdoor attacks on large language models",
       "authors": ["Yige Li"],
       "year": 2024,
       "arxiv_id": "2408.12798",
       "relevance": "Provides the benchmark framework and Sleeper implementation used as a baseline in this paper."
     },
     {
       "title": "A survey on large language model (LLM) security and privacy: The good, the bad, and the ugly",
       "authors": ["Yifan Yao"],
       "year": 2023,
       "arxiv_id": "2312.02003",
       "relevance": "Broad survey on LLM security threats including jailbreak and backdoor attacks, providing threat landscape context."
     },
     {
       "title": "Llama Guard: LLM-based input-output safeguard for human-AI conversations",
       "authors": ["Hakan Inan"],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "The safety judge model (LLaMA-Guard-3-8b) used as the core component of bi-GRPO's rule-based reward system and for ASR evaluation."
     }
   ]
 }