ai-research-survey

scan.json (30184B)

---

 {
   "paper": {
     "title": "System Prompt Poisoning: Persistent Attacks on Large Language Models Beyond User Injection",
     "authors": [
       "Zongze Li",
       "Jiawei Guo",
       "Haipeng Cai"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2505.06493",
     "doi": "10.48550/arXiv.2505.06493"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "System prompt poisoning is highly effective, reducing reasoning model accuracy to near-zero (<4%) on MATH in stateless API scenarios. The attack persists across 500-turn conversations without significant weakening for reasoning models. User-side augmentation techniques (ICL, CoT) and the Explicit Reminder defense fail to mitigate the attack. Brute-force poisoning is extremely cheap (~2 seconds, <1.3k tokens) while adaptive strategies cost orders of magnitude more but offer only marginally different effectiveness.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository or archive is linked anywhere in the paper. The Auto-SPP framework is described algorithmically (Algorithm 1, Appendix B) but no implementation is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available MATH dataset (Hendrycks et al., 2021) and HumanEval dataset (Chen et al., 2021). Both are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details are provided. The paper does not describe software versions, libraries, or runtime environments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While the experimental setup is described at a high level in Section 5.2, there are no scripts, commands, or detailed instructions for replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 1 and throughout the paper are point estimates (e.g., '0.8%', '99.1%↓') with no confidence intervals or error bars reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used despite making many comparative claims (e.g., 'reasoning models are significantly more vulnerable'). All comparisons are based on raw numerical differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage decreases from baseline are reported for all results (e.g., '99.1%↓' in Table 1), providing effect size context relative to the unpoisoned baseline."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '500 randomly selected samples' from MATH but provides no justification for this number and no power analysis. HumanEval uses its standard 164 problems without discussion of adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. All results appear to be from single experimental runs with no indication of result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Each configuration includes a 'No poisoning' baseline (Table 1) showing model accuracy without attack, against which all poisoning strategies are compared."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The defense baseline (Explicit Reminder, Yi et al. 2025) is contemporary. The models tested (GPT-5-mini, Gemini-2.5-flash, GPT-4o-mini) are current. For an attack paper, the 'no poisoning' baseline is the natural comparator."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "While three distinct poisoning strategies are compared, there is no ablation study examining which components of each strategy drive effectiveness (e.g., number of exemplars, reasoning chain length, or specific instruction formulations)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Each domain uses only a single metric: solution accuracy for MATH and pass@1 for HumanEval. No secondary metrics (e.g., output quality beyond correctness, attack detectability) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. Human inspection of poisoned outputs (e.g., to assess stealthiness or output quality beyond correctness) would be relevant but is absent."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The attacks are generated independently of the test data. The poisoned system prompts are crafted without access to the specific MATH or HumanEval test examples, and evaluation is on the standard benchmarks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides detailed breakdowns by attack scenario (4 types), poisoning strategy (3 types), model (4 models), and domain (MATH vs HumanEval). RQ2-RQ5 provide further breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E discusses GPT-3.5-turbo as a failure case where attacks did not significantly degrade performance and in some cases paradoxically improved it (Tables 3-4). The paper identifies this as a limitation."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The GPT-3.5-turbo results (Tables 3-4 in Appendix E) show the attack failing or even improving model performance. The paper explicitly notes 'performance paradoxically improved after poisoning' in some cases."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims of high effectiveness, persistence, robustness against CoT, and stealthiness against defenses are all supported by corresponding experimental sections (RQ1-RQ4) with quantitative results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that poisoning causes performance degradation is supported by controlled experiments: the only variable changed between conditions is the system prompt (poisoned vs unpoisoned), with the same user inputs and models."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Persistent Attacks on Large Language Models' broadly, but testing covers only 4 closed-source models and 2 domains. Appendix E acknowledges limitations to open-source models and other domains, but the main text and abstract make broad generalizations (e.g., 'persistently impacts all subsequent user interactions')."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper offers interpretations for observed patterns (e.g., why reasoning models are more vulnerable) but does not consider alternative explanations. For instance, it does not discuss whether simple distractor text in the system prompt (without malicious intent) would cause similar degradation, or whether the effect is due to instruction hierarchy rather than poisoning per se."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy and pass@1 and frames these as 'task performance degradation,' which matches the measurement granularity. No proxy gap exists between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names only: 'Gemini-2.5-flash', 'GPT-5-mini', 'GPT-4o-mini'. No snapshot dates, API versions, or specific model identifiers are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figures 2-4 show illustrative examples of poisoning on emotion classification tasks, but the actual system prompts and poisoned prompts used for MATH and HumanEval experiments are not provided. The reader cannot reconstruct the exact prompts sent to the models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens, etc.) are reported for any of the model API calls, despite these significantly affecting output behavior."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The experiments involve direct API calls and simulated interactive sessions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '500 randomly selected samples' from MATH but does not describe the random selection procedure (seed, stratification, difficulty distribution). For interactive scenarios, conversation history summarization is mentioned but not detailed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E contains a dedicated 'Limitations' section with three substantive subsections discussing model dependency, scope of models/domains, and scope of defenses."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E discusses specific threats: (1) GPT-3.5-turbo anomaly showing attacks depend on model reasoning ability, (2) limited to 4 closed-source models excluding open-source LLMs, (3) only one defense mechanism tested."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E explicitly states: 'The findings may not generalize to the full spectrum of LLMs, particularly open-source models,' 'our experiments were conducted on tasks from the mathematics and coding domains,' and 'Our stealthiness evaluation (RQ4) was limited to a single, common black-box defense mechanism.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, individual predictions, poisoned prompts used) is released. Only aggregated accuracy numbers are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.2 provides a high-level description of the experimental procedure but lacks detail on the random selection of MATH samples, interactive conversation simulation mechanics, and how context summarization was handled when exceeding token limits."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (MATH, HumanEval)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from benchmark selection through poisoned prompt generation to evaluation is described at a high level, but key transformation steps are missing: how interactive conversations were simulated, what context summarization threshold was used, and how pass@1 was computed for HumanEval."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are listed with their affiliation at the University at Buffalo, Department of Computer Science and Engineering. No conflict with the evaluated products (Google, OpenAI models)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of the funder cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four models tested. This is relevant because baseline performance on MATH and HumanEval could be inflated by contamination."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MATH or HumanEval problems appeared in any model's training data. Both benchmarks were published in 2021 and are widely known."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MATH (2021) and HumanEval (2021) are both published benchmarks that predate all tested models' training data. No contamination analysis or discussion is provided despite this being relevant to interpreting baseline scores."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The ethical statement notes all experiments were in controlled environments using public datasets."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports Auto-SPP framework costs (time and tokens for generating poisoned prompts), but the cost of running the full experimental evaluation (thousands of API calls across 4 models × 4 scenarios × 3 strategies × 2 datasets) is not reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Only the Auto-SPP generation costs are stated (Table 2). The total computational budget for the full experimental evaluation (API costs, compute hours) is not disclosed."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs producing each result is never stated. It is unclear whether results are from single runs or averaged over multiple trials."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The number of exemplars in ICL/CoT strategies, prompt formulations tried, and helper LLM settings are not discussed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports results for all three strategies across all four scenarios without selecting a single 'best' configuration. All results are shown comprehensively in Table 1."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so correction for multiple comparisons is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own attack framework (Auto-SPP) and implement the defense baseline (Explicit Reminder) themselves, without acknowledging potential bias from author-evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While Table 2 reports Auto-SPP costs and Table 1 reports effectiveness, performance is not explicitly analyzed as a function of compute budget. The tradeoff is discussed qualitatively but not presented as a performance-cost curve."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether accuracy on MATH and pass@1 on HumanEval are valid measures of the real-world impact of system prompt poisoning. No discussion of whether these benchmarks capture the threat model's practical concerns."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The experiments use direct API calls to models."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. MATH (2021) and HumanEval (2021) both predate the tested models' training periods. Models may have seen benchmark solutions during training, affecting baseline scores."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. No analysis of whether the evaluation setup leaks information beyond what would be available in real-world attack scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. No discussion of whether MATH or HumanEval problems may share structural similarities with training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used. No canary strings, membership inference, or decontamination procedures are described."
       }
     }
   },
   "claims": [
     {
       "claim": "System prompt poisoning drastically reduces model accuracy, with reasoning models collapsing to near-zero (<4%) on MATH in stateless API scenarios.",
       "evidence": "Table 1 shows Gemini-2.5-flash drops from 93.2% to 0.8% (99.1%↓) under brute-force in Explicit+API, and GPT-5-mini from 91.4% to 2.2% (97.6%↓) under Adaptive ICL. All three strategies achieve >96% degradation for reasoning models in API scenarios.",
       "supported": "strong"
     },
     {
       "claim": "Reasoning models are significantly more vulnerable to SPP than non-reasoning models.",
       "evidence": "Table 1 consistently shows reasoning models degrading to <4% while non-reasoning models retain 25-62% accuracy across scenarios. However, no statistical test supports the 'significantly' claim.",
       "supported": "moderate"
     },
     {
       "claim": "Poisoning effects persist across 500-turn conversations without significant weakening for reasoning models.",
       "evidence": "Figure 5 shows accuracy remains suppressed below 15% for reasoning models across 100, 300, and 500 turns in both interactive scenarios. Lines are 'nearly flat.'",
       "supported": "moderate"
     },
     {
       "claim": "User-side prompting augmentation techniques (ICL, CoT) are ineffective at mitigating system prompt poisoning.",
       "evidence": "Figure 6 shows accuracy remains below 10% for all three augmentation methods when poisoning is active, tested on Gemini-2.5-flash in Explicit+API on MATH. However, this is tested on only one model and one dataset.",
       "supported": "moderate"
     },
     {
       "claim": "The Explicit Reminder defense is completely ineffective against SPP.",
       "evidence": "Figure 7 shows near-zero accuracy persists when Explicit Reminder is applied, tested on Gemini-2.5-flash in both Explicit and Implicit API scenarios on MATH. Tested on only one model.",
       "supported": "moderate"
     },
     {
       "claim": "Auto-SPP can automate poisoning with brute-force being extremely cheap (~2 seconds, <1.3k tokens).",
       "evidence": "Table 2 reports brute-force requiring 1.9-2.2 seconds and 0.7-1.3k tokens, versus 268-340 seconds and 123-247k tokens for adaptive strategies.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance reporting",
       "detail": "All results in Table 1 and throughout the paper are point estimates from apparently single runs. With no variance reporting, it is impossible to assess result stability or whether observed differences are meaningful."
     },
     {
       "flag": "Limited defense evaluation",
       "detail": "Only one defense (Explicit Reminder) is tested, and it is the simplest possible defense. More sophisticated defenses (input filtering, adversarial training, prompt sandboxing) are discussed only theoretically in Section 6. The stealthiness claim is based on defeating one naive defense."
     },
     {
       "flag": "RQ3 and RQ4 tested on single model only",
       "detail": "The robustness (RQ3) and stealthiness (RQ4) experiments use only Gemini-2.5-flash on MATH. These single-model results are used to make general claims about attack robustness and defense ineffectiveness."
     },
     {
       "flag": "No actual experiment prompts provided",
       "detail": "The poisoned system prompts used for MATH and HumanEval experiments are never shown. Figures 2-4 show illustrative examples from a different task (emotion classification). Without the actual prompts, the experiments cannot be replicated."
     },
     {
       "flag": "Model versions unspecified",
       "detail": "Only marketing names are used (Gemini-2.5-flash, GPT-5-mini, GPT-4o-mini). Without snapshot dates or API versions, results may not be reproducible as model behavior changes across versions."
     }
   ],
   "cited_papers": [
     {
       "title": "Ignore previous prompt: Attack techniques for language models",
       "authors": ["Fábio Perez", "Ian Ribeiro"],
       "year": 2022,
       "arxiv_id": "2211.09527",
       "relevance": "Foundational work on prompt injection attacks against LLMs, which SPP extends to the system prompt layer."
     },
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra", "Christoph Endres", "Thorsten Holz", "Mario Fritz"],
       "year": 2023,
       "relevance": "Extended prompt injection to indirect attacks on LLM-integrated applications, directly relevant to supply-chain attack vectors discussed in the threat model."
     },
     {
       "title": "FATH: Authentication-based test-time defense against indirect prompt injection attacks",
       "authors": ["Jiongxiao Wang", "Fangzhou Wu", "Wendi Li", "Jinsheng Pan", "Edward Suh", "Z Morley Mao", "Muhao Chen", "Chaowei Xiao"],
       "year": 2024,
       "arxiv_id": "2410.21492",
       "relevance": "Proposes a defense against prompt injection that SPP's threat model targets; relevant to understanding the defense landscape."
     },
     {
       "title": "Adaptive attacks break defenses against indirect prompt injection attacks on LLM agents",
       "authors": ["Qiusi Zhan", "Richard Fang", "Henil Shalin Panchal", "Daniel Kang"],
       "year": 2025,
       "arxiv_id": "2503.00061",
       "relevance": "Demonstrates that adaptive attacks bypass all existing prompt injection defenses, complementing SPP's findings on defense ineffectiveness."
     },
     {
       "title": "InjecAgent: Benchmarking indirect prompt injections in tool-integrated large language model agents",
       "authors": ["Zhixiang Zhan", "Zifan Ying", "Daniel Kang"],
       "year": 2024,
       "arxiv_id": "2403.02691",
       "relevance": "Benchmark framework for evaluating LLM agent vulnerabilities to indirect prompt injection."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini", "Milad Nasr", "J Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Foundational jailbreaking work demonstrating universal adversarial attacks on LLMs, relevant to the broader attack surface taxonomy."
     },
     {
       "title": "Data poisoning in LLMs: Jailbreak-tuning and scaling laws",
       "authors": ["Dillon Bowen", "Brendan Murphy", "Will Cai", "David Khachaturov", "Adam Gleave", "Kellin Pelrine"],
       "year": 2024,
       "arxiv_id": "2408.02946",
       "relevance": "Studies data poisoning in LLMs at the training level, complementing SPP's inference-time poisoning approach."
     },
     {
       "title": "PoisonBench: Assessing large language model vulnerability to data poisoning",
       "authors": ["Tingchen Fu", "Mrinank Sharma", "Philip Torr", "Shay B Cohen", "David Krueger", "Fazl Barez"],
       "year": 2024,
       "arxiv_id": "2410.08811",
       "relevance": "Benchmark for LLM data poisoning vulnerability assessment, relevant to understanding poisoning attack evaluation methodology."
     },
     {
       "title": "Baseline defenses for adversarial attacks against aligned language models",
       "authors": ["Neel Jain", "Avi Schwarzschild", "Yuxin Wen", "Gowthami Somepalli", "John Kirchenbauer", "Ping-yeh Chiang", "Micah Goldblum", "Aniruddha Saha", "Jonas Geiping", "Tom Goldstein"],
       "year": 2023,
       "arxiv_id": "2309.00614",
       "relevance": "Proposes baseline defenses including perplexity-based detection for adversarial inputs to LLMs."
     },
     {
       "title": "Benchmarking and defending against indirect prompt injection attacks on large language models",
       "authors": ["Jingwei Yi", "Yueqi Xie", "Bin Zhu", "Emre Kiciman", "Guangzhong Sun", "Xing Xie", "Fangzhao Wu"],
       "year": 2025,
       "relevance": "Source of the 'Explicit Reminder' defense evaluated in RQ4, directly relevant to SPP's defense bypass claims."
     },
     {
       "title": "On the (in)security of LLM app stores",
       "authors": ["Xinyi Hou", "Yanjie Zhao", "Haoyu Wang"],
       "year": 2024,
       "arxiv_id": "2407.08422",
       "relevance": "Studies security of LLM application ecosystems, directly relevant to SPP's passive distribution threat model via app stores."
     },
     {
       "title": "PoisonedRAG: Knowledge corruption attacks to retrieval-augmented generation of large language models",
       "authors": ["Wei Zou", "Runpeng Geng", "Binghui Wang", "Jinyuan Jia"],
       "year": 2024,
       "arxiv_id": "2402.07867",
       "relevance": "Attacks RAG knowledge bases to corrupt LLM outputs, a related poisoning vector targeting a different component of the LLM pipeline."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Security researchers and LLM application developers can use the threat model and attack strategies to red-team their systems, though no code is released."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The instruction hierarchy bias (system prompts override user prompts) is somewhat expected; the magnitude and persistence of the effect across 500 turns adds novelty but is not deeply contrarian."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Demonstrates a novel persistent attack vector that bypasses existing defenses, degrades reasoning models to near-zero accuracy, and can be automated cheaply."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "No major controversy or dramatic claims about specific companies, though it implies current defenses are fundamentally inadequate."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released. The Auto-SPP framework is described algorithmically only."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "University at Buffalo researchers (not a famous AI lab), though the paper tests well-known models (GPT-5, Gemini, GPT-4o)."
     }
   }
 }