ai-research-survey

scan-v4.json (34161B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "PromptScreen: Efficient Jailbreak Mitigation Using Semantic Linear Classification in a Multi-Staged Pipeline",
     "authors": [
       "Akshaj Prashanth Rao",
       "Advait Singh",
       "Saumya Kumaar Saksena",
       "Dhruv Kumar"
     ],
     "year": 2025,
     "venue": "Unknown",
     "arxiv_id": "2512.19011",
     "doi": "10.48550/arXiv.2512.19011"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 93.4% accuracy, 96.5% specificity, and 10× lower latency than ShieldGemma are directly supported by Tables 2 and 3. The '35.1% to 93.4%' improvement and '≈450s to 47s' TTC claims match the tabulated results.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('LSVM improves accuracy,' 'this module achieves...'). The ablation study (Section 6) and pipeline configuration comparisons (Table 3) provide controlled manipulation of components, supporting these claims through single-variable ablation design.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'scalable, resource-efficient protection against prompt-based adversarial behavior' broadly, but the system is tested only on English prompts with specific attack types from known datasets. While Section 8 acknowledges multilingual limitations, the title and conclusion make sweeping claims about 'modern LLM-driven applications' that exceed the tested scope.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For instance: could ShieldGemma's poor 35.1% accuracy be due to misconfiguration? Could the dataset composition favor keyword-based detection? Could the 0% ASR reflect the attack corpus rather than the defense quality? None of these alternatives are considered.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures classification accuracy, ASR, and TTC — the same quantities it claims to evaluate. There is no proxy gap: the paper claims to measure defense accuracy and latency, and that is what it measures.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 ('Limitations') is a dedicated limitations section with three specific subsections: Multilingual Constraints, Contextual and Multi-turn Awareness, and Generalization to Zero-Day Attacks.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: (1) preprocessing pipeline is optimized for English only, (2) evaluates single-turn prompts and may miss multi-turn distributed attacks, (3) SVM performance bounded by training corpus diversity for zero-day attacks. These are concrete, study-specific threats.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states the system 'does not address or evaluate protection against multilingual jailbreak attempts,' is limited to 'isolated, single-turn inputs,' and performance is 'bounded by the diversity of its training corpus' for novel attack strategies.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The Acknowledgement section (Section 9) only mentions the use of ChatGPT for writing assistance, with no mention of grants, sponsors, or funding agencies.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: BITS Pilani and Trustwise. Dhruv Kumar has a dual affiliation with both institutions.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot assess funder independence since funding is not disclosed. One author (Dhruv Kumar) is affiliated with Trustwise, a company in the AI trust/safety space that could benefit commercially from demonstrating lightweight defense effectiveness.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper. The Trustwise affiliation raises potential commercial interest in the defense pipeline's success, but this is not discussed.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined: 'jailbreak' (inputs coercing model to violate safety), 'prompt-injection' (inputs targeting application logic), 'ASR' (attack success rate fraction). Taxonomy is explicit in dataset section.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions are explicitly stated: (1) efficient LSVM semantic filter, (2) multi-stage configurable pipeline, (3) 30,000+ labeled prompt dataset. Each is clearly framed as a contribution.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 'Related Work' discusses four prior areas (detection/mitigation, token manipulation, prompt optimization, benchmarks) and explicitly positions this work: 'In contrast, our work treats defense as configurable, multi-stage pipeline.' Shows differentiation.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "GitHub repository URL provided in Section 7: https://github.com/dronefreak/PromptScreen. The paper states 'the source code is available at' this URL.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 states: 'Both the corpus and associated preprocessing scripts are included in the open-source release of this work to facilitate continued benchmarking and community evaluation.' The dataset of 30,937 labeled prompts is described as released.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, conda environment file, or detailed environment specification is mentioned in the paper. Library versions are not listed.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions, README with commands, or 'Reproducing Results' section is included in the paper. The algorithms are described in pseudocode but no executable reproduction guide is provided.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results in Tables 2, 3, and 4 are reported as point estimates only (e.g., '93.40% accuracy'). No confidence intervals, error bars, or ± notation appears anywhere.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "The paper claims the SVM 'outperforms' ShieldGemma and other baselines based solely on comparing raw metric values. No statistical significance tests (p-values, t-tests, bootstrap tests, etc.) are used.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports improvements with baseline context: 'improves overall accuracy from 35.1% to 93.4%' and 'reducing average time-to-completion from ≈450s to 47s, yielding over 10× lower latency.' Tables 2 and 3 provide sufficient context to assess magnitude.",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "No justification given for the corpus size of 30,937 prompts or the test set of 2,000+ prompts. No power analysis or discussion of whether the sample size is adequate for the claims being made.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "All results appear to be from single runs. No standard deviations, variance across seeds, or multi-run spread measures are reported in any table.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 compares three defense approaches: ShieldGemma, the classifier cluster, and the LSVM. Table 3 compares three pipeline configurations with different defense subsets.",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "ShieldGemma (Google, 2024) is a recent production-grade defense. The classifier cluster uses contemporary HuggingFace models (textdetox/xlmr-large-toxicity-classifier-v2, jackhhao/jailbreak-classifier).",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 6 presents a systematic SVM ablation study across 8 feature configurations (word n-grams, char n-grams, hybrid). Table 3 also ablates pipeline configurations by including/excluding defense stages.",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Tables 2 and 4 report precision, sensitivity, specificity, negative predictive value, and accuracy. Table 3 adds block rate, ASR, and average TTC. Well over two metrics are used.",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "All evaluation is automated. Classification performance uses standard metrics; ASR is judged by an LLM-as-a-judge and verified by Gemini. No human evaluation of the system's outputs or classifications is performed.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 states: 'A held-out test set of more than 2,000 prompts is reserved exclusively for evaluation of defense configurations and Attack Success Rate. No prompt appears in more than one split.'",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 breaks down performance per individual defense module. Table 3 shows results per pipeline configuration. Table 4 shows per-SVM-configuration results. Table 1 shows dataset composition by class (jailbreak, benign, prompt-injection).",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.1 explicitly discusses 'defenses that were implemented but did not meet our robustness or usability requirements,' including the heuristic vector analyzer (false positive issues) and PPA (insufficient protection). Section 8 discusses failure scenarios.",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.1 reports two failed defense strategies: the heuristic vector analyzer ('prone to false positives on benign prompts') and Polymorphic Prompt Assembly ('protection was not sufficient against the strongest attacks').",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Specific model identifiers are provided: ShieldGemma-2B, gpt-oss:20b (target LLM), textdetox/xlmr-large-toxicity-classifier-v2, jackhhao/jailbreak-classifier. These are sufficiently specific model identifiers to reproduce the setup.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "ShieldGemma is described as consuming 'the prompt alongside structured safety guidelines' but these guidelines are not shown. The LLM-as-a-judge Attack Evaluator uses 'predefined criteria' that are not provided. No actual prompt text is given for any LLM-based component.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "No hyperparameters are reported for the Linear SVM (C value, regularization), TF-IDF vectorizer (max_features, min/max df), VectorDB similarity threshold τ, or any model component. The paper describes the architecture but not its parameterization.",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used. The system is a sequential classification pipeline, not an agent with tool use, memory, or iterative reasoning.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.3.1 documents the text preprocessing pipeline in detail: lowercasing, emoji-to-text conversion, punctuation removal, tokenization, alphabetic filtering, stopword removal, and POS-aware WordNet lemmatization. Dataset construction is documented in Section 3.2.",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 states: 'Both the corpus and associated preprocessing scripts are included in the open-source release.' All dataset instances are 'serialized in structured JSON with explicit source metadata and taxonomy labels.'",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 describes data sources in detail: manually curated jailbreaks from prior literature, automated adversarial prompts from ADV-LLM, prompt-injection examples from gentelbench-v1, and benign prompts from public instruction-following datasets. Labeling taxonomy with three classes is defined.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": true,
           "answer": true,
           "justification": "Data sources are described: adversarial prompts from prior academic literature (Liu et al., Sun et al., Pathade), ADV-LLM automated outputs, gentelbench-v1 dataset, and public instruction-following datasets for benign prompts. Each source's purpose and origin is stated in Section 3.2.",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": false,
           "justification": "While data sources and final composition (Table 1: 30,937 total) are described, the pipeline from source collection to final dataset lacks detail. How many examples came from each source is not stated. The filtering process ('manually reviewed to remove ambiguous cases') gives no counts of how many were removed or what criteria defined 'ambiguous.'",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": false,
           "answer": false,
           "justification": "The paper tests defense mechanisms (SVM, classifier cluster, etc.) against adversarial prompts, not a pre-trained model's capability on a benchmark. The target LLM (gpt-oss:20b) is used only to measure ASR (whether attacks pass through defenses), not to evaluate model knowledge.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": false,
           "answer": false,
           "justification": "The paper evaluates defense tools rather than a pre-trained model's capability on a benchmark. Contamination in the traditional sense (model has seen test data during pre-training) does not apply.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": false,
           "answer": false,
           "justification": "The paper tests defense mechanisms, not model capability on benchmarks. The SVM is trained by the authors on their own dataset, not a pre-trained model being evaluated on a potentially contaminated benchmark.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study. All evaluation is automated on a curated prompt dataset.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. The study evaluates defense mechanisms on text prompts without involving human subjects.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in the study.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Time-to-Classify (TTC) is reported in Table 3: {VectorDB, Classifier} at 30.13s, {YARA, Cluster, VectorDB, ShieldGemma} at 450.35s, and {SVM, VectorDB, Classifier} at 47.24s. Section 5.3 discusses computational efficiency as a primary evaluation dimension.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No total computational budget is stated. Hardware specifications, GPU/CPU used for experiments, total training time for the SVM, or total evaluation time across the test set are not reported.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "No mention of random seeds or seed sensitivity analysis. The SVM training and all evaluations appear to be single-run results with no assessment of seed-dependent variance.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": false,
           "justification": "The number of experimental runs is never stated. Results appear to come from single runs but this is not explicitly confirmed.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "No hyperparameter search budget is reported. The SVM ablation (Section 6) tests different feature configurations but the number of configurations tried, search method, or computational cost of the search is not disclosed.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": false,
           "justification": "The main results (Table 2) use the LSVM module but it is unclear which specific TF-IDF configuration was used — the ablation (Table 4) shows char n-gram (2,4) as best at 94.09%, but the main results report 93.4%. The selection of the final configuration is not justified.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": true,
           "answer": false,
           "justification": "Multiple comparisons are made across defense configurations (Table 2, 3) and SVM variants (Table 4) without any correction for multiple comparisons. No statistical tests are performed at all, let alone corrected ones.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The authors train, configure, and evaluate their own SVM system against baselines without acknowledging author-evaluation bias. No independent evaluation is performed, and no discussion of the bias inherent in self-evaluation appears.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 reports both defense effectiveness (block rate, ASR) and TTC per configuration, enabling performance-vs-compute comparison. Section 5.3 explicitly discusses the latency trade-off between lightweight (SVM, 47s) and heavyweight (ShieldGemma, 450s) defenses.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether its curated corpus of 30,937 prompts is representative of real-world attack distributions, whether the attack difficulty matches production threats, or whether the benchmark actually measures what is claimed about 'robust defense.'",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is involved. The defense pipeline is a sequential classifier, and comparisons are between defense modules, not between models in different scaffolds.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether adversarial prompts in the training set were generated or collected before or after the test prompts, or whether temporal ordering could introduce leakage in the SVM's training.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether the evaluation setup provides information not available in real deployment. The VectorDB stores known attack signatures that could overlap structurally with test attacks from the same source campaigns.",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The paper states 'No prompt appears in more than one split' but does not address whether train and test prompts from the same source (e.g., ADV-LLM) share structural patterns or were generated by the same attack algorithms, which would compromise independence.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": false,
           "justification": "No concrete leakage detection or prevention method is described. Only basic non-overlap between splits is ensured, without n-gram analysis, membership inference, or decontamination pipelines.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Semantic LSVM classifier achieves 93.4% accuracy and 96.5% specificity on jailbreak/prompt-injection detection",
       "evidence": "Table 2 reports Text processing + Semantic LSVM with accuracy 0.9340, specificity 0.9650 on held-out test set",
       "supported": "strong"
     },
     {
       "claim": "LSVM-based configuration improves over ShieldGemma baseline by 58 percentage points in accuracy (35.1% → 93.4%)",
       "evidence": "Table 2 shows ShieldGemma accuracy 0.3513 vs LSVM 0.9340 on same test set",
       "supported": "strong"
     },
     {
       "claim": "Full multi-stage pipeline achieves 0% Attack Success Rate (blocks all 1,456 adversarial test prompts)",
       "evidence": "Table 3, {SVM, VectorDB, Classifier} configuration shows 0.00% ASR with 1405 blocked prompts, 51 attempted, 0 successful",
       "supported": "strong"
     },
     {
       "claim": "Pipeline reduces inference latency 10× compared to ShieldGemma (47s vs 450s)",
       "evidence": "Table 3 shows avg TTC of 47.24s for {SVM, VectorDB, Classifier} vs 450.3459s for {YARA, Cluster, VectorDB, ShieldGemma}",
       "supported": "strong"
     },
     {
       "claim": "Character n-gram features outperform word-level features in SVM ablation study",
       "evidence": "Table 4: char n-gram (2,4) achieves 94.09% accuracy vs baseline word unigram at 90.27%",
       "supported": "strong"
     },
     {
       "claim": "System effectively handles emoji, Unicode homoglyph, and whitespace obfuscation attacks",
       "evidence": "Ablation study (Section 6) augmented dataset with leetspeak, homoglyphs, whitespace perturbations and tested robustness; char n-grams designed to capture these",
       "supported": "moderate"
     },
     {
       "claim": "Staged architecture with early-stage semantic filter enables high accuracy with minimal latency overhead",
       "evidence": "SVM alone achieves 0.47s avg TTC (Table 3); full pipeline 47.24s. Design discussion (Section 3.1) explains early rejection reduces downstream processing",
       "supported": "moderate"
     },
     {
       "claim": "Simple Linear SVM outperforms complex model-based approaches for prompt-level security",
       "evidence": "LSVM (Table 2: 93.4% acc) outperforms ShieldGemma (35.1%) and Classifier cluster (65.0%); however, comparison not controlled for model capacity or training data differences",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "systems-evaluation"
   ],
   "key_findings": "PromptScreen combines classical machine learning (Linear SVM with TF-IDF features) and modular defenses into a multi-stage pipeline for detecting prompt injection and jailbreak attacks. The semantic LSVM component achieves 93.4% accuracy and 96.5% specificity, substantially outperforming the model-based ShieldGemma baseline (35.1% accuracy) while reducing inference latency from ~450s to 47s. When combined with complementary defenses (vector similarity matching, classifier ensembles), the full system achieves 0% Attack Success Rate on the evaluation set of 30,000+ labeled prompts while maintaining negligible false-positive rates on benign inputs. The results suggest that efficient, interpretable defenses can be more effective than heavyweight LLM-based moderators for real-time security at scale.",
   "red_flags": [
     {
       "flag": "No confidence intervals or variance reporting",
       "detail": "All results (Tables 2-3) report single-point values without CIs, std devs, or ranges. No multiple runs or cross-validation reported, limiting statistical rigor."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Large differences between methods (e.g., LSVM 93.4% vs ShieldGemma 35.1%) not tested for significance. Unclear if differences could arise by chance."
     },
     {
       "flag": "Limited system ablation",
       "detail": "Feature-level ablation provided (Section 6) but no system-level ablation. Cannot isolate contribution of each pipeline stage (SVM vs VectorDB vs Classifier cluster)."
     },
     {
       "flag": "LLM-as-a-judge prompts not disclosed",
       "detail": "ASR evaluation uses unspecified LLM prompts and safety guidelines. Reproducibility of Attack Evaluator verdicts is limited without exact prompt text."
     },
     {
       "flag": "Hyperparameters not reported",
       "detail": "SVM hyperparameters (C, kernel, tolerance) absent. VectorDB similarity threshold τ not specified. Reproducibility compromised."
     },
     {
       "flag": "Single target model tested",
       "detail": "Evaluation against only 'gpt-oss:20b.' Generalization to other models (GPT-4, Claude, Llama) unknown."
     },
     {
       "flag": "Dataset sourced from known attacks only",
       "detail": "Training/test data aggregated from academic literature and documented exploits. May not represent novel zero-day attack space despite Section 8 acknowledging this limitation."
     },
     {
       "flag": "Acknowledged English-only preprocessing",
       "detail": "System does not support multilingual jailbreaks (Section 8, Limitation 1). Limits applicability to non-English deployments."
     },
     {
       "flag": "Acknowledged single-turn limitation",
       "detail": "Pipeline evaluates isolated prompts, not multi-turn dialogue. Real adversaries exploit conversation history (Section 8, Limitation 2). Results may not translate to agentic systems."
     },
     {
       "flag": "Potential dataset overfitting",
       "detail": "VectorDB trained on same 28,000 prompts used for SVM training. Though train/test separated, both models learned on same distribution. Robustness to truly novel attacks uncertain."
     }
   ],
   "cited_papers": [
     {
       "title": "Jailbreaking ChatGPT via Prompt Engineering: An Empirical Study",
       "relevance": "Empirical taxonomy of jailbreak strategies; dataset source for adversarial prompts"
     },
     {
       "title": "Greedy Coordinate Gradient-Based Search for Universal Adversarial Attacks",
       "relevance": "Adversarial suffix optimization techniques; attack vectors evaluated in defense pipeline"
     },
     {
       "title": "Prompt Injection Attacks and Defenses in Vision-Language Models",
       "relevance": "Defines prompt injection vs jailbreaking distinction; related defense architecture patterns"
     },
     {
       "title": "Emoji Attack: A Method for Misleading Judge LLMs in Safety Risk Detection",
       "relevance": "Token-level obfuscation attacks; motivates character-level feature engineering in SVM"
     },
     {
       "title": "Tree of Attacks: Jailbreaking Black-Box LLMs Automatically",
       "relevance": "Automated black-box jailbreak search methods; adversarial corpus source"
     },
     {
       "title": "Intention Analysis Makes LLMs a Good Jailbreak Defender",
       "relevance": "Pre-generation intent analysis approach; compared baseline moderation strategy"
     },
     {
       "title": "Attention Tracker: Detecting Prompt Injection Attacks in LLMs",
       "relevance": "Model-internal defense signals; alternative detection mechanism not pursued here"
     },
     {
       "title": "From LLMs to MLLMs to Agents: A Survey of Emerging Security Challenges",
       "relevance": "Agentic attack surface expansion; motivates need for efficient early-stage defenses"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Proposes a deployable defense pipeline with released code, useful for practitioners securing LLM applications, though not a plug-and-play solution."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that a simple SVM outperforms an LLM-based moderator is somewhat surprising but aligns with known advantages of classical ML for specific classification tasks."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Directly addresses LLM jailbreak and prompt injection attacks, a growing security concern for AI-powered applications."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicit criticism of ShieldGemma's performance (35.1% accuracy) could generate mild discussion, but no dramatic controversy."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and dataset released on GitHub, allowing practitioners to test and replicate the defense pipeline."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors are from BITS Pilani and Trustwise, neither of which are widely recognized in the AI safety community."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }