ai-research-survey

scan.json (34727B)

---

 {
   "paper": {
     "title": "When Reject Turns into Accept: Quantifying the Vulnerability of LLM-Based Scientific Reviewers to Indirect Prompt Injection",
     "authors": [
       "Devanshu Sahoo",
       "Manish Prasad",
       "Vasudev Majhi",
       "Jahnvi Singh",
       "Vinay Chamola",
       "Yash Sinha",
       "Murari Mandal",
       "Dhruv Kumar"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2512.10449",
     "doi": "10.48550/arXiv.2512.10449"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Adversarial invisible-text injections in PDFs can flip LLM reviewer decisions from Reject to Accept at rates up to 86.26% for open-source models, with token-level obfuscation (Class I) strategies proving most effective. Proprietary models like GPT-5 show near-perfect robustness, but distilled variants (GPT-5-Mini) and advanced reasoning models (Gemini-2.5-Pro) exhibit specific vulnerability patterns — a 'safety tax' from distillation and a 'reasoning trap' where instruction-following fidelity is weaponized. Social engineering attacks (Class III) frequently backfire, actually reducing scores in both open and closed-source models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract provides a repository URL (https://anonymous.4open.science/r/llm-jailbreak-FC9E/) and Section 1 Contribution 4 states 'we will release our complete experimental framework including code and dataset.' The anonymous link is functional at time of review."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the dataset of 200 papers and injection framework will be released via the anonymous repository. The ICLR 2025 OpenReview papers used are publicly available. Section 1 Contribution 4 explicitly commits to releasing the dataset."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is provided. The paper mentions Ollama for local models and MinerU for PDF conversion but provides no version pinning or dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. The pipeline is described at a high level (Section 3.3, Figure 1) but there are no README-style commands or scripts to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates — average score increases (Figures 2, 3), percentage changes in acceptance rates (Table 3), and WAVS scores (Figures 4, 5). No confidence intervals, error bars, or uncertainty measures are provided anywhere."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'Cls1MSM achieved near-perfect score inflation on mistral-small:22b (+13.95)' vs. qwen3:30b resilience) without any statistical significance tests. All comparisons are based on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as Average Score Increase on a 0-35 scale (e.g., +13.95 for Cls1MSM on mistral-small), percentage increase in acceptance rates (e.g., 86.26%), and the WAVS composite metric. Baselines are provided for context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset comprises 200 papers (30 templates, 125 rejected, 30 poster, 15 spotlight) with a subset of 50 for closed-source models 'to accommodate cost and rate limits' (Section 4.1). No power analysis or statistical justification for these sizes is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregate averages are reported. No standard deviations, interquartile ranges, or variance across papers within each (model, strategy) condition. The strip plots in Appendix G show individual scores but no summary spread measures are computed."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "All metrics are computed relative to the un-injected baseline. Section 3.3 describes how the original paper is scored without adversarial injection, and all score increases (Figures 2, 3) and decision flip rates (Table 3) are measured against this baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models evaluated are contemporary (GPT-5, Claude Haiku 4.5, Gemini 2.5 Pro/Flash, DeepSeek-R1-32B, Qwen3-30B, etc.) and represent the state of the art as of late 2025. The baseline is the same models without attack, which is appropriate."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "While 15 strategies across 3 classes are tested individually, there is no systematic ablation of attack components (e.g., testing misspellings alone vs. variable injection alone within Cls1MSM, or testing invisible text vs. visible injection). The strategies are tested as fixed units."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three primary metrics are used: Average Score Increase (Section 4.3), Percentage Increase in Acceptance Rates (Table 3), and the novel WAVS metric (Appendix A) with its three sub-components (Score Sensitivity, Semantic Flip Severity, Risk Alignment)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. There is no human assessment of whether the LLM reviews are actually valid, no human judges evaluating the quality of the adversarial injections, and no comparison of LLM review outputs against human expert reviews."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "All 200 papers are used for evaluation with no held-out set. The subset of 50 used for closed-source models was selected for cost reasons, not as a held-out validation split. No dev/test separation is described."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per-model and per-strategy in heatmaps (Figures 2, 3), per attack class in the taxonomy (Classes I-III), and decomposed by WAVS components (Figure 7). Table 3 provides per-model per-strategy acceptance rate changes. Appendix D-G provides additional breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses the 'Backfire Effect' where Class III strategies trigger score decreases (Section 5.1), shows specific failure examples (falcon3 penalizing Cls3SP by -4.07), and Appendix C provides a case study of both a vulnerable model (mistral-small) and a robust model (gemini-2.5-flash) refusing the attack."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: Social engineering attacks frequently backfire (Section 5.1, 5.2); GPT-5 shows near-zero vulnerability across all strategies; qwen3:30b demonstrates resilience despite being a large model; many strategy-model combinations show negative score changes in Table 3."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'decision flip rates of up to 86.26% in open-source models' — supported by Table 3 (mistral-small Cls1DRA: 86.26%). Claims about 'reasoning traps in proprietary systems' are supported by Section 5.2's analysis of Gemini-2.5-Pro. The 15 strategies and 13 models are documented."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims ('adversarial injections manipulate scores') are justified by the controlled experimental design: the same papers are scored with and without injection, isolating the causal effect of the injection. This is a valid single-variable manipulation design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Limitations) explicitly bounds generalization: CS-specific dataset may not generalize to humanities/clinical sciences, assumes 'Lazy Reviewer' worst case, proprietary model results are a 'snapshot in time,' and only textual/layout injections tested (not multimodal)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative factors: model size does not linearly predict vulnerability (Section 5.1), distillation effects explain GPT-5-Mini vulnerability (Section 5.2), tokenization robustness explains proprietary model resilience (Section 5.2 point 3), and tulu3:8b's 'mode collapse' failure mode is identified."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The WAVS metric (Appendix A) explicitly distinguishes between the proxy (score inflation) and the outcome (decision flips). The three-component decomposition separates Score Sensitivity, Semantic Flip Severity, and Risk Alignment, acknowledging that not all score increases equally represent vulnerability."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Most model versions are specified only by marketing name: 'GPT-5', 'GPT-5-Mini', 'Gemini 2.5 Flash', 'Gemini 2.5 Pro' without API version or snapshot date. The exception is Claude Haiku 4.5 which includes a date (claude-haiku-4-5-20251001). Open-source models list sizes but not commit hashes or specific checkpoints."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The system prompt (Listing 2) and user prompt (Listing 3) are provided in full in Appendix C. The 15 adversarial strategy prompts are documented in Table 4 (Appendix B) with mechanism descriptions and truncated but substantial prompt snippets. Listing 1 shows a full injection payload."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other inference hyperparameters are reported for any of the 13 models. These settings significantly affect LLM output and vulnerability to adversarial manipulation."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The pipeline is a linear processing chain (PDF → MinerU → injection → API call → regex parsing) without agentic loops, tool use, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 documents the preprocessing pipeline: MinerU converts PDFs to layout-preserving Markdown, adversarial prompts are appended in white 1pt font, and the pipeline stages are detailed in Figure 1 and Appendix C. Section 4.1 describes the dataset composition and sourcing."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 is a dedicated 'Limitations' section with substantive discussion spanning four specific limitations: dataset domain specificity, Lazy Reviewer assumption, API snapshot volatility, and exclusion of multimodal attacks."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 lists specific threats: 'our evaluation is bounded by the size and domain specificity of our dataset; we utilized 200 manuscripts primarily sourced from Computer Science venues'; proprietary models 'are accessed via black-box APIs subject to continuous, unannounced updates'; the Lazy Reviewer assumption 'represents a worst-case security posture.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states what was NOT tested: other disciplines ('humanities or clinical sciences'), human-in-the-loop scenarios, and 'multi-modal adversarial attacks embedded within scientific figures or charts, which remains an open avenue for future investigation.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper commits to releasing 'our complete experimental framework including code and dataset' (Section 1 Contribution 4) and provides an anonymous repository link. The strip plots in Appendix G show per-paper raw scores, and the full raw data is stated to be part of the release."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes data collection: 200 papers from two sources — official conference templates (IEEE, ACL) and ICLR 2025 OpenReview real-world submissions, balanced across accepted (Spotlight/Poster) and rejected manuscripts with specific counts (30 templates, 125 rejected, 30 poster, 15 spotlight)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes paper sourcing: official conference templates serve as vacuous baselines, and real-world papers come from the ICLR 2025 OpenReview track. The rationale for each source is explained (templates test catastrophic failure, real papers test real-world effectiveness)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The five-stage pipeline is documented in Section 3.3 and Figure 1: (1) Attack Injection with invisible white-font text, (2) MinerU PDF-to-Markdown conversion, (3) System/User prompt construction, (4) Multi-provider inference loop, (5) Regex JSON parsing with failure logging. Appendix C provides a detailed case study walkthrough."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. Section 9 (Acknowledgement) only mentions 'the use of ChatGPT in improving the presentation and grammar.' No grants, sponsors, or funding agencies are listed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: BITS Pilani and KIIT University. The authors are academic researchers not affiliated with any of the model providers being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making independence impossible to assess. Since the absence of funding disclosure is itself a transparency issue, this cannot be marked as satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests adversarial robustness of LLM reviewers to prompt injection attacks, not model knowledge or capability on a benchmark. The evaluation measures whether injected prompts can manipulate review scores, which is independent of training data contamination."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale: this is a red-teaming study measuring adversarial manipulation, not model capability on a knowledge benchmark. Train/test overlap is not the relevant threat to validity here."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale: the paper evaluates defense robustness against adversarial attacks, not pre-trained model knowledge. Contamination of the review papers in training data would affect baseline scores but not the adversarial manipulation effect being measured."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved. The study evaluates LLM models on adversarial paper variants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, tokens consumed, or latency figures are reported despite running 13 models × (200 or 50) papers × 16 conditions (original + 15 strategies) = tens of thousands of inference calls."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours for local model inference (8 models via Ollama), API spending for proprietary models (5 models), or total wall-clock time are reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or stochastic variation analysis. Each (model, paper, strategy) triplet appears to be evaluated once, with no assessment of run-to-run variability."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many times each inference triplet was executed. Section 3.3 describes iterating through triplets but does not specify whether each is run once or multiple times."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The adversarial prompts are fixed, and no tuning of injection placement, font size, or prompt phrasing variations is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All 15 strategies are reported across all 13 models — there is no selective reporting of best configurations. The full heatmaps (Figures 2, 3), Table 3, and Appendix G present complete results."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 13 models × 15 strategies = 195 conditions with many implicit comparisons, but no statistical tests are performed at all, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the 15 attack strategies and evaluate their effectiveness. No acknowledgment of author-evaluation bias or independent validation by third parties is provided."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison of vulnerability profiles as a function of compute budget. Open-source models (8B-32B) are compared with proprietary models of unknown size, but no compute-normalized analysis is performed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether its evaluation setup (LLM scoring papers on a 0-35 rubric) actually measures real-world vulnerability of the review process. The Lazy Reviewer assumption is stated but not validated against actual reviewer behavior."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The same evaluation pipeline (MinerU conversion → system prompt → user prompt with injection) is used across all models, controlling for scaffold effects. The only variation is the inference provider (Ollama vs. API), which is inherent to the model access method."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The ICLR 2025 papers could have appeared in the training data of models like GPT-5 or Gemini 2.5 Pro. If a model has seen a paper's reviews or acceptance status during training, baseline scores would be confounded."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The evaluation provides full paper text to the model, which is realistic, but the paper does not consider whether template papers are trivially identifiable to models that have seen conference templates in training."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Multiple papers from the same venue (ICLR 2025) may share structural similarities. The 30 official conference templates likely share formatting that could bias model responses in correlated ways."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, temporal splits, or decontamination measures are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "Token-level obfuscation strategies (Class I) are the most universally potent attack vectors, with Maximum Mark Magyk (Cls1MSM) achieving +13.95 score inflation on mistral-small:22b and Disguise and Reconstruction (Cls1DRA) achieving +13.87 on the same model.",
       "evidence": "Figure 2 heatmap shows these values. Section 5.1 discusses the 'Dominance of Token-Level Obfuscation' with specific score increases per model.",
       "supported": "strong"
     },
     {
       "claim": "Adversarial injections can achieve decision flip rates of up to 86.26% in open-source models (mistral-small with Cls1DRA strategy).",
       "evidence": "Table 3 reports percentage increase in acceptance rates. Mistral-small Cls1DRA shows 86.26% increase. Multiple open-source models show >50% flip rates.",
       "supported": "moderate"
     },
     {
       "claim": "Model vulnerability does not correlate linearly with parameter count — qwen3:30b is resilient while gemma3:27b is catastrophically vulnerable.",
       "evidence": "Section 5.1 discusses 'Scale-Independent Vulnerability'. Figure 2 shows qwen3:30b scores between -0.82 and +2.254 while gemma3:27b reaches +13.75. Figure 5 shows their relative vulnerability rates.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-5 exhibits near-perfect robustness with negligible score inflation across all attack vectors and zero decision flips.",
       "evidence": "Figure 3 shows GPT-5 as 'almost entirely dark blue'. Table 3 shows 0.00% flip rate across all 15 strategies for GPT-5. Figure 5 places GPT-5 at 0.0 relative vulnerability.",
       "supported": "strong"
     },
     {
       "claim": "A 'safety tax' exists from model distillation — GPT-5-Mini is significantly more vulnerable than GPT-5, with susceptibility to Logic Decipherer (Cls2LDA, +1.84) and Evidence-Based Persuasion (Cls3EBP, +1.60).",
       "evidence": "Section 5.2 compares GPT-5 vs GPT-5-Mini. Figure 3 heatmap shows the contrast. However, Table 3 shows GPT-5-Mini has 0.00% flip rate across all strategies, suggesting the score inflation doesn't translate to actual decision flips.",
       "supported": "weak"
     },
     {
       "claim": "Social engineering attacks (Class III) frequently trigger the 'Backfire Effect', causing negative score adjustments instead of inflation.",
       "evidence": "Section 5.1 reports falcon3 penalizing Social Proof (Cls3SP) by -4.07, gemma3 penalizing Expert Endorsement by -3.95. Section 5.2 reports GPT-5 (-0.96), GPT-5-Mini (-0.89), and Claude-Haiku (-0.66) all penalizing Cls3SP.",
       "supported": "strong"
     },
     {
       "claim": "Advanced reasoning capabilities can become a vulnerability vector — Gemini-2.5-Pro shows the highest proprietary vulnerability spike against Symbolic Masking and Context Redirection (Cls1SMCR, +2.54).",
       "evidence": "Section 5.2 discusses the 'Reasoning Trap' hypothesis. Figure 3 shows the +2.54 spike. However, Table 3 shows Gemini-2.5-Pro has limited actual decision flips (13.04% for Cls1SMCR).",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single-run point estimates with no confidence intervals, error bars, standard deviations, or repeated trials. LLM outputs are stochastic — a single inference per (model, paper, strategy) triplet means reported scores could be noise rather than signal, especially for small effect sizes in proprietary models."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "Claims like 'qwen3:30b demonstrated remarkable resilience' vs. 'gemma3:27b was catastrophically vulnerable' and the entire Safety Gap / Reasoning Trap narrative rest on comparing raw numbers without any significance testing. With 200 papers, formal tests would be feasible."
     },
     {
       "flag": "Missing hyperparameters for inference",
       "detail": "Temperature, top-p, and max_tokens are not reported for any model. Temperature alone can dramatically affect LLM susceptibility to adversarial manipulation. Results may not be reproducible without these settings."
     },
     {
       "flag": "Reduced proprietary model dataset without representativeness analysis",
       "detail": "Closed-source models are evaluated on only 50 of 200 papers 'to accommodate cost and rate limits' (Section 4.1). No analysis of whether this subset is representative of the full dataset, and no stratified sampling description. This makes open-source vs. closed-source comparisons asymmetric."
     },
     {
       "flag": "Safety tax claim not supported by decision flips",
       "detail": "The 'safety tax' claim about GPT-5-Mini is based on score inflation (+1.84), but Table 3 shows GPT-5-Mini has 0.00% decision flip rate across ALL strategies — identical to GPT-5. The framing overstates the practical impact of small score increases that never change decisions."
     },
     {
       "flag": "Adversarial strategy design not independently validated",
       "detail": "The authors both designed the 15 attack strategies and evaluated their effectiveness. No independent red-team, external validation, or comparison against independently designed attacks is provided. Author-evaluation bias (Lucic et al. 2018) could inflate apparent effectiveness."
     }
   ],
   "cited_papers": [
     {
       "title": "Not what you've signed up for: Compromising real-world llm-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "relevance": "Foundational work formalizing indirect prompt injection against LLMs processing external content, directly relevant to the survey's coverage of prompt injection attacks."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini", "Milad Nasr", "J. Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Demonstrates universal adversarial suffixes for jailbreaking aligned LLMs, a key attack technique in the LLM safety landscape."
     },
     {
       "title": "JailbreakBench: an open robustness benchmark for jailbreaking large language models",
       "authors": ["Patrick Chao", "Edoardo Debenedetti", "Alexander Robey"],
       "year": 2024,
       "relevance": "Standardized benchmark for evaluating LLM jailbreak robustness, relevant to the survey's coverage of adversarial evaluation methodologies."
     },
     {
       "title": "BadJudge: Backdoor vulnerabilities of LLM-as-a-judge",
       "authors": ["Terry Tong", "Fei Wang", "Zhe Zhao", "Muhao Chen"],
       "year": 2025,
       "arxiv_id": "2503.00596",
       "relevance": "Directly relevant to surveying vulnerabilities in LLM-as-a-judge systems, examining backdoor attack vectors."
     },
     {
       "title": "How johnny can persuade LLMs to jailbreak them: Rethinking persuasion to challenge AI safety by humanizing LLMs",
       "authors": ["Yi Zeng", "Hongpeng Lin", "Jingwen Zhang", "Diyi Yang", "Ruoxi Jia", "Weiyan Shi"],
       "year": 2024,
       "relevance": "Demonstrates that persuasion techniques can jailbreak LLMs, relevant to the survey's coverage of social engineering attacks against AI systems."
     },
     {
       "title": "Play guessing game with LLM: Indirect jailbreak attack with implicit clues",
       "authors": ["Zhiyuan Chang", "Mingyang Li", "Yi Liu", "Junjie Wang", "Qing Wang", "Yang Liu"],
       "year": 2024,
       "relevance": "Token-level obfuscation jailbreak technique relevant to the survey's adversarial attack taxonomy."
     },
     {
       "title": "Investigating the Vulnerability of LLM-as-a-Judge Architectures to Prompt-Injection Attacks",
       "authors": ["Narek Maloyan", "Bislan Ashinov", "Dmitry Namiot"],
       "year": 2025,
       "relevance": "Directly investigates prompt injection against LLM-as-a-judge, closely related to the survey's scope on adversarial manipulation of AI evaluation systems."
     },
     {
       "title": "Prompt injection attacks on llm generated reviews of scientific publications",
       "authors": ["Janis Keuper"],
       "year": 2025,
       "arxiv_id": "2509.10248",
       "relevance": "Prior work on prompt injection against LLM-based scientific reviewers using naive direct injection, which this paper explicitly extends with more sophisticated attacks."
     },
     {
       "title": "A wolf in sheep's clothing: Generalized nested jailbreak prompts can fool large language models easily",
       "authors": ["Peng Ding", "Jun Kuang", "Dan Ma"],
       "year": 2024,
       "relevance": "Nested jailbreak prompts technique directly adapted in this paper's Scenario Nesting (Cls2SN) attack strategy."
     },
     {
       "title": "FlipAttack: Jailbreak LLMs via flipping",
       "authors": ["Yue Liu", "Xiaoxin He", "Miao Xiong"],
       "year": 2025,
       "relevance": "Flip-based jailbreak technique adapted as Cls2FA (Flip Attack) in this paper's taxonomy."
     },
     {
       "title": "Making them ask and answer: jailbreaking large language models in few queries via disguise and reconstruction",
       "authors": ["Tong Liu", "Yingjie Zhang", "Zhe Zhao"],
       "year": 2024,
       "relevance": "Disguise and reconstruction jailbreak technique adapted as Cls1DRA, the most effective open-source attack in this paper."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "One of the evaluated reasoning models, relevant to the survey's coverage of LLM capabilities and robustness."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly relevant to conference organizers and reviewers deploying AI review tools, but the attacks require PDF manipulation expertise and the defenses are high-level recommendations."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The 'reasoning trap' finding — that more capable models can be MORE vulnerable to certain attacks — challenges the assumption that scale improves safety."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Demonstrates that invisible PDF injections can flip Reject-to-Accept at 86% rates, directly threatening the integrity of scientific peer review at major conferences."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "The 'Lazy Reviewer' framing, ICLR/NeurIPS as targets, and the 'meritocratic collapse' narrative create controversy around AI use in peer review."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Anonymous repository link provided but requires setting up multiple LLM APIs and local model infrastructure to reproduce."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Tests well-known models (GPT-5, Claude, Gemini, DeepSeek) and references major conferences (NeurIPS, ICLR, AAAI)."
     }
   }
 }