ai-research-survey

scan.json (33636B)

---

 {
   "paper": {
     "title": "Proactive Hardening of LLM Defenses with HASTE",
     "authors": [
       "Henry Chen",
       "Victor Aranda",
       "Samarth Keshari",
       "Ryan Heartfield",
       "Nicole Nichols"
     ],
     "year": 2026,
     "venue": "NDSS Symposium 2026, LAST-X Workshop",
     "arxiv_id": "2601.19051",
     "doi": "10.48550/arXiv.2601.19051"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "HASTE's most aggressive hard-negative mining with semantic fuzzing (HM-Max+Sem) reduces baseline detector accuracy from 95.91% to 31.76%, demonstrating that iterative hard-negative generation is far more effective at producing evasive prompts than basic fuzzing alone. HASTE-retrained models recover to ~94% accuracy on held-out evaluation data, with hard-mining configurations achieving this in 50% fewer iteration loops than baseline strategies. Semantic fuzzing is the most effective individual technique, and combining all fuzzing types can paradoxically counteract semantic fuzzing's evasive effectiveness. Most improvement plateaus after the first or second iteration, suggesting diminishing returns from extended iteration loops.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper. The generation prompts are explicitly withheld: 'These specific prompts are not provided to prevent their misuse' (Section III-B)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The seed dataset of ~4,500 adversarial prompts is described as from 'internal and public sources' and 'proprietary internal findings' (Section IV-B) but is not released. No download link or dataset archive is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or library version specifications are provided. The paper names models used (GPT-4o, DeBERTa-v3, JailJudge, T5-base paraphraser) but provides no environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided. The framework architecture is described conceptually but cannot be reproduced without code, prompts, and data."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A mentions 'overlapping 95% confidence intervals of the sampling estimates (e.g., It2: 0.6257–0.6794 vs. It5: 0.6417–0.6757)' to support the claim that post-iteration-2 fluctuations are not statistically meaningful."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No formal significance tests (p-values, t-tests, etc.) are reported. Comparative claims like 'HM-Max-Sem is the most effective strategy' (Section V-B) are based on comparing point estimates without statistical testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout, e.g., 'dropping classifier accuracy from 95.91% to 37.00%' (Section V-B), delta improvements in Tables V and VI (e.g., '+19.9' for Benign), and percentage drops from baseline in Table III."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 500 prompts per run, ~4,500 seed prompts, 1,000 hold-out set, and 10 iterations is stated but never justified. No power analysis or rationale for these specific numbers is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Tables II, III, and IV report single numbers per configuration per iteration with no standard deviation, variance across runs, or indication of how many runs produced each number. The confidence intervals in Section V-A appear to be single-sample binomial intervals, not cross-run variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The Base configuration (no hard-mining, no fuzzing) serves as a clear baseline. Multiple configurations are systematically compared against it in Tables II, III, and IV."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The base detection model is ProtectAI/deberta-v3-base-prompt-injection (2023), a contemporary prompt-injection detector. The judge model JailJudge (2024) and target model GPT-4o are also recent."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The experimental design is structured as a systematic ablation. Section IV-A notes: 'the experiment configurations implicitly implement module ablations by toggling specific parameters.' Fuzzing variants, hard-mining ratios, and combinations are independently toggled across 11 configurations (Table I)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two distinct metrics are used: (i) iteration accuracy measuring baseline detector performance on generated prompts, and (ii) HASTE-optimized model accuracy (H accuracy) measuring retrained detector performance on the held-out evaluation set (Section IV-A)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed. Evaluation relies entirely on automated metrics: the DeBERTa classifier for detection and JailJudge (an LLM) for prompt potency scoring. No human review of generated adversarial prompts or detection quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A clearly states: 'an evaluation set is held out and remains untouched by any fuzzing, synthetic generation or hard negative mining.' The 1,000-prompt held-out set is used for Table IV H accuracy results, independent of in-loop training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables V and VI provide per-category accuracy breakdowns (Benign, Obfuscation, Objective Manipulation, Other, Role Play) for each configuration on the out-of-loop evaluation set."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A discusses how combining all fuzzing types counteracts semantic fuzzing's effectiveness. SHAP analysis (Figure 9) examines token-level attributions for false-positive examples. Section V-B notes that HM-Bal shows proportional rather than synergistic gains."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: Base configuration 'produces little to no impact on the maliciousness of generated samples' (Section V-A); combining all fuzzing types counteracts semantic fuzzing's evasive effectiveness; '10 iterations are not necessary' (Section V-B); benchmark sensitivity may be insufficient to capture robustness differences (Section V-C)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claim of '~64% reduction in malicious prompt detection' is supported by Table III (HM-Max-Sem: 95.91% → 31.76%). The claim of 'significantly fewer iteration loops' is supported by Table IV showing HM configurations reach ~93% at M5 vs Base requiring M10."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('hard negative mining successfully evades baseline detectors,' 'HASTE optimizes detection efficacy') are supported by a controlled ablation design where configurations differ by one or two parameters (Table I), isolating the contribution of fuzzing and hard-mining."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title ('LLM Defenses') and abstract ('optimize LLM defense') frame results broadly, but experiments use only one detector (ProtectAI DeBERTa-v3), one target model (GPT-4o), one judge (JailJudge), and one seed dataset. The Future Work section acknowledges the need to 'evaluate multiple base classifier architectures' (Section VI), confirming the limitation without bounding it in claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper offers one interpretation for each result without systematically considering alternatives. For example, the observation that combined fuzzing counteracts semantic fuzzing is noted as 'beyond the scope of this study' (Section V-A) rather than explored. No confound analysis or robustness checks against alternative explanations are provided."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures detector accuracy directly and frames it as such — 'iteration accuracy' and 'H accuracy' are clearly defined as detection metrics (Section IV-A). The paper does not over-claim beyond what the metrics measure, though it uses broader framing in the title and abstract."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'GPT-4o' as target and generator, 'JailJudge' as judge, 'ProtectAI/deberta-v3-base-prompt-injection' as detector, and 'humarin/chatgpt_paraphraser_on_T5_base' for semantic fuzzing. However, no API snapshot dates or version identifiers are provided for GPT-4o, which changes behavior across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly withholds prompts: 'These specific prompts are not provided to prevent their misuse' (Section III-B). Only natural-language descriptions of what generation strategies do are given."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Some parameters are stated — 5% syntactic mutation rate, maliciousness threshold of 2, 500 prompts per run, 80/20 train/test split. However, critical hyperparameters for GPT-4o generation (temperature, top-p) and DeBERTa fine-tuning (learning rate, epochs, batch size, optimizer) are not reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The HASTE pipeline is a data generation and model training framework, not agentic scaffolding with tools, retry logic, or memory/context management. The LLMs are used as generation and evaluation components in a batch pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B documents the pipeline: seed dataset partitioned into 3,500 for generation and 1,000 hold-out, prompts deduplicated and standardized, labeled with taxonomy metadata, balanced 50/50 with benign corpus (~40,000 benign examples), and split 80/20 for train/test with stratification."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. Section VI ('Future Work') discusses areas for improvement but frames them as expansion opportunities rather than honest discussion of what the current work does not show."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The Future Work section mentions that 'current benchmarks may not be sensitive enough' (Section VI) but does not identify specific threats to the internal or external validity of the current experiments."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The Future Work section implicitly acknowledges gaps (single detector architecture, single target model, fixed fuzzing parameters) but these are framed as expansion opportunities rather than explicit scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (seed prompts, generated adversarial prompts, evaluation scores, training data) is made available. The seed dataset includes 'proprietary internal findings' (Section III-A) which precludes release."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes the seed dataset: '~4,500 prompts aggregated from internal and public sources' including 'real-world jailbreaks observed in the wild, public benchmark datasets, proprietary internal findings, and crowd-sourced CTF-style challenges.' Prompts were 'deduplicated and standardized' and labeled by attack taxonomy."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. The data consists of adversarial and benign prompts from existing datasets and internal sources."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline stages are described conceptually with figures (Figures 1, 3-8), but exact counts at each filtering stage are not provided. For example, how many prompts passed the evaluation threshold (score ≥ 2), how many survived deduplication, and how many fuzzed variants were produced per iteration are not reported."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section is present. All five authors are employed by Palo Alto Networks, a major cybersecurity company, but no funding statement is included."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as Palo Alto Networks employees with @paloaltonetworks.com email addresses in the paper header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Palo Alto Networks is a cybersecurity company that sells LLM security products. The company has a direct financial interest in demonstrating that proactive defense hardening frameworks like HASTE are effective. The funder (employer) is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosure, or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests prompt-injection detection models, not a pre-trained model's knowledge or capability on a benchmark. The evaluation is about whether a classifier can identify adversarial prompts, not whether an LLM has memorized benchmark answers."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper evaluates defense detection systems rather than pre-trained model capability on benchmarks. Standard train/test separation is used for the classifier but benchmark contamination in the traditional sense does not apply."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper tests defense detection efficacy, not pre-trained model knowledge. Contamination of the LLM's training data with benchmark answers is not the relevant concern here."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or latency figures are reported despite extensive use of GPT-4o for generation, evaluation, and labeling across 10 iterations of 11 configurations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total API spend, hardware specifications, or training time for DeBERTa fine-tuning are reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed experiments are reported. Tables II, III, and IV show single numbers per configuration per iteration with no indication of variability across random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs producing each result is not stated. It is unclear whether each table entry represents a single run or an average across multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported for DeBERTa fine-tuning or for tuning the HASTE pipeline parameters. The evaluation threshold is set to 2 based on a single observation ('65% of generated samples scored 1') without systematic search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All 11 configurations are reported transparently in Tables II-IV rather than cherry-picking the best. The paper presents the full comparison grid, allowing readers to assess each configuration's relative merit."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No formal statistical tests are used, so no multiple comparison correction is applied. With 11 configurations compared across 10+ iterations, the risk of spurious differences is high but unaddressed."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own HASTE framework against their own baseline configurations using their own seed dataset and pipeline. No acknowledgment of self-comparison bias or independent evaluation is provided."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Different configurations require different amounts of compute (e.g., fuzzing adds processing, 10 iterations vs 5), but performance is not reported as a function of compute budget. The 50% reduction in iterations is framed as efficiency but not quantified in compute terms."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The Future Work section briefly notes that 'current benchmarks may not be sensitive enough to capture the deeper robustness learned from these harder samples' (Section VI), but no analysis of construct validity is performed. Whether iteration accuracy and H accuracy truly measure 'robustness' and 'evasiveness' is not examined."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The HASTE pipeline configurations ARE the variable being studied, not a confound. The detection model is held constant (DeBERTa-v3) across all comparisons, and the pipeline differences are the intentional experimental manipulation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4o's training data might contain similar adversarial prompts to those in the seed set, which could affect generation quality. The held-out evaluation set addresses train/test leakage for the classifier but temporal leakage for the generative components is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides information that would not be available in real deployment. For example, whether the LLM-as-a-Judge scores leak signal about prompt structure back into the pipeline is not analyzed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The held-out evaluation set and in-loop training data are drawn from the same ~4,500 prompt seed dataset. Whether these two partitions are truly independent in distribution (e.g., same authors, same attack patterns) is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method (canary strings, membership inference, n-gram overlap, decontamination pipeline) is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Hard negative mining with semantic fuzzing reduces baseline detector accuracy from 95.91% to approximately 31.76% (a ~64% reduction).",
       "evidence": "Table III shows HM-Max+Sem iteration accuracy dropping from 95.91% (It0) to 31.76% (It10). The abstract reports 'approximately 64%' reduction.",
       "supported": "strong"
     },
     {
       "claim": "HASTE-optimized models achieve comparable final accuracy (~93-94%) with 50% fewer iteration loops than baseline strategies.",
       "evidence": "Table IV shows HM configurations reaching ~93% H accuracy at M5, while the Base configuration only reaches 81.06% at M5 and needs M10 to reach 92.80%. Section V-C states 'at least a 50% reduction of the number of HASTE iteration loops.'",
       "supported": "strong"
     },
     {
       "claim": "Semantic fuzzing is the most effective individual fuzzing method for producing evasive prompts.",
       "evidence": "Table II shows Base+Sem reducing accuracy to 65.26% vs 71.62% (Syntactic), 70.86% (Format), and 70.37% (All). Section V-A concludes Base+Sem 'comparatively produced the largest percentage of prompts to successfully evade detection.'",
       "supported": "strong"
     },
     {
       "claim": "Most adversarial generation improvement plateaus after the first or second iteration.",
       "evidence": "Tables II and III show the largest accuracy drops at It1, with minimal further degradation through It10. Section V-A notes 'the peak improvement for producing evasive malicious prompts typically occurs in the second iteration' and Section V-B states 'the utility of all strategies plateaus almost immediately after the first iteration.'",
       "supported": "strong"
     },
     {
       "claim": "Combining all fuzzing types (semantic + syntactic + format) counteracts the evasive effectiveness of semantic fuzzing alone.",
       "evidence": "Table II: Base+All achieves 70.37% accuracy vs Base+Sem at 65.26%. Section V-A states 'the addition of syntactic and format fuzzing may counteract the evasive effectiveness of semantic fuzzing.' SHAP analysis (Figure 9) shows syntactic changes dominate token attributions.",
       "supported": "moderate"
     },
     {
       "claim": "The HASTE framework provides a self-improving mechanism that makes detectors increasingly robust over time.",
       "evidence": "Table IV shows H accuracy improving from 82.12% (M0) to 93-94% (M10) across configurations. However, 'self-improving' is somewhat overclaimed — the improvement plateaus quickly and the framework is tested on a single detector with a single seed dataset.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own framework",
       "detail": "All five authors are Palo Alto Networks employees evaluating a Palo Alto Networks security framework. No competing interests statement is provided, and the company has direct financial interest in demonstrating effective LLM defense hardening."
     },
     {
       "flag": "Prompts and code withheld — reproduction impossible",
       "detail": "Generation prompts are explicitly withheld ('to prevent misuse'), no source code is released, and the seed dataset includes proprietary data. Independent reproduction of the experimental results is not possible."
     },
     {
       "flag": "Single detector architecture",
       "detail": "All experiments use only ProtectAI DeBERTa-v3 as the base detector. The paper's broad claims about 'LLM Defenses' rest entirely on one model, undermining generalizability. The Future Work section acknowledges the need to 'evaluate multiple base classifier architectures.'"
     },
     {
       "flag": "No variance or multi-run reporting",
       "detail": "All tables report single point estimates per configuration per iteration. Without variance across seeds or multiple runs, it is impossible to assess whether observed differences between configurations are reliable or within noise."
     },
     {
       "flag": "No cost reporting despite heavy API usage",
       "detail": "The pipeline uses GPT-4o extensively for generation, evaluation (target model), and labeling across 11 configurations × 10 iterations, but reports zero cost information. Practitioners cannot assess whether HASTE is economically viable."
     }
   ],
   "cited_papers": [
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["A. Zou", "Z. Wang", "N. Carlini", "M. Nasr", "J. Z. Kolter", "M. Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Foundational work on automatically generated adversarial suffixes that reliably bypass LLM alignment, directly motivating the need for frameworks like HASTE."
     },
     {
       "title": "Attack and defense techniques in large language models: A survey and new perspectives",
       "authors": ["Z. Liao", "K. Chen", "Y. Lin", "K. Li", "Y. Liu", "H. Chen", "X. Huang", "Y. Yu"],
       "year": 2025,
       "arxiv_id": "2505.00976",
       "relevance": "Comprehensive survey of LLM attacks and defenses that categorizes adversarial prompts, optimization attacks, and application-level threats."
     },
     {
       "title": "Efficient Detection of Toxic Prompts in Large Language Models",
       "authors": ["Y. Liu", "J. Yu", "H. Sun", "L. Shi", "G. Deng", "Y. Chen", "Y. Liu"],
       "year": 2024,
       "doi": "10.1145/3691620.3695018",
       "relevance": "Introduces ToxicDetector, an embedding-based approach to toxic prompt detection that represents the progression from textual to hybrid detection models."
     },
     {
       "title": "Llama Guard: LLM-based input-output safeguard for human-AI conversations",
       "authors": ["H. Inan", "K. Upasani", "J. Chi"],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "Foundational LLM safety guardrail model that frames prompt safety as structured classification, one of the detection models the HASTE framework aims to improve upon."
     },
     {
       "title": "NeMo Guardrails: A toolkit for controllable and safe LLM applications with programmable rails",
       "authors": ["T. Rebedea", "R. Dinu", "M. Sreedhar", "C. Parisien", "J. Cohen"],
       "year": 2023,
       "arxiv_id": "2310.10501",
       "relevance": "Programmable runtime guardrail framework implementing dialogue control flows and safe tool-use routing for LLM applications."
     },
     {
       "title": "JailbreakBench: An open robustness benchmark for jailbreaking large language models",
       "authors": ["P. Chao", "E. Debenedetti", "A. Robey"],
       "year": 2024,
       "arxiv_id": "2404.01318",
       "relevance": "Standardized benchmark for evaluating jailbreak effectiveness with consistent evaluation and cross-model comparison."
     },
     {
       "title": "HarmBench: A standardized evaluation framework for automated red teaming and robust refusal",
       "authors": ["M. Mazeika", "L. Phan", "X. Yin", "A. Zou"],
       "year": 2024,
       "arxiv_id": "2402.04249",
       "relevance": "Standardized red-teaming benchmark focusing on harmful behaviors, providing automated evaluation of LLM safety."
     },
     {
       "title": "A comprehensive survey in LLM(-agent) full stack safety: Data, training and deployment",
       "authors": ["K. Wang"],
       "year": 2025,
       "arxiv_id": "2504.15585",
       "relevance": "Comprehensive survey covering the full stack of LLM safety from data to deployment, relevant to understanding the defense landscape HASTE targets."
     },
     {
       "title": "JailGuard: A universal detection framework for LLM prompt-based attacks",
       "authors": ["X. Zhang", "C. Zhang", "T. Li"],
       "year": 2025,
       "arxiv_id": "2312.10766",
       "relevance": "Universal detection framework for prompt-based attacks that demonstrates detection models struggle with out-of-distribution attacks — a problem HASTE addresses."
     },
     {
       "title": "AID: Adaptive integration of detectors for safe AI with language models",
       "authors": ["X. Wang", "E. Diao", "Q. Le", "J. Ding", "A. Anwar"],
       "year": 2025,
       "relevance": "Demonstrates that combining multiple detectors improves adaptability over single-model approaches, providing context for HASTE's single-detector evaluation limitation."
     },
     {
       "title": "Building guardrails for large language models",
       "authors": ["Y. Dong", "R. Mu", "G. Jin"],
       "year": 2024,
       "arxiv_id": "2402.01822",
       "relevance": "Describes multi-layered safety architectures for LLMs that HASTE's detection hardening component fits within."
     },
     {
       "title": "JailJudge: A comprehensive jailbreak judge benchmark with multi-agent enhanced explanation evaluation framework",
       "authors": ["F. Liu", "Y. Feng", "Z. Xu"],
       "year": 2024,
       "arxiv_id": "2410.12855",
       "relevance": "The LLM-as-a-judge model used in HASTE's evaluation stage for scoring prompt-response pair maliciousness."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The HASTE concept is directly applicable for security teams hardening LLM defenses, but no code, prompts, or data are released for practitioners to actually use."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Results largely confirm expected patterns (hard-negative mining helps, iterations plateau); the finding that combined fuzzing counteracts semantic fuzzing is mildly surprising."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates that adversarial prompts can reduce detection model accuracy from 96% to 32%, highlighting the vulnerability of current prompt-injection detectors."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenge to specific products or companies, no dramatic narrative."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released. Prompts explicitly withheld."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Palo Alto Networks is well-known in cybersecurity but not a top-tier AI research lab in public perception."
     }
   }
 }