ai-research-survey

scan.json (34937B)

---

 {
   "paper": {
     "title": "Sentra-Guard: A Multilingual Human-AI Framework for Real-Time Defense Against Adversarial LLM Jailbreaks",
     "authors": [
       "Md. Mehedi Hasan",
       "Ziaur Rahman",
       "Rafid Mostafiz",
       "Md. Abir Hossain"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.22628",
     "doi": "10.48550/arXiv.2510.22628"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Sentra-Guard proposes a hybrid jailbreak detection system combining SBERT-FAISS retrieval, fine-tuned DistilBERT classification, zero-shot NLI (BART-MNLI), and human-in-the-loop feedback. The paper claims 99.98% accuracy and 0.004% ASR on HarmBench-28K, with 47ms average latency. However, the paper has severe integrity issues: numerous references are grossly misattributed (cited content does not match the actual referenced papers), results are internally inconsistent (abstract vs body numbers differ), and the dataset size is contradictory (D2 stated as excluded from training, then reported as included).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section V (Ethical Considerations) states 'we will release a redacted version of Sentra-Guard's source code' — this is a promise of future release, not an actual release. No repository URL is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The primary dataset D1 (HarmBench-28K) is described as an 'open-access adversarial benchmark.' Cross-dataset evaluation uses JailbreakV-28K, JBB-Behaviors, and JailbreakTracer, which are also public. However, the auxiliary dataset D2 is described only as 'publicly available red-teaming repositories' without specific identification."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section IV.A mentions PyTorch, HuggingFace Transformers, Tesla T4 GPU (8GB), Apple M1 CPU, and specific model names (distilbert-base-uncased, facebook/bart-large-mnli), but no version numbers for any library, no requirements.txt, no Dockerfile, and no reproducible environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the general pipeline (Algorithm 1) but gives no scripts, commands, or README for replicating results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables III-VI are point estimates only. No confidence intervals, error bars, or uncertainty measures are reported for any metric (accuracy, F1, ASR, latency)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Sentra-Guard outperforms baselines (Static Keyword Filter, Zero-Shot Classifier, Ensemble Filter) but provides no statistical significance tests. Comparisons are based solely on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute performance differences are reported with baseline context: ASR of 0.004% vs LlamaGuard-2 (1.3%) and OpenAI Moderation (3.7%). Table III shows accuracy of 99.98% vs 92.83% (ensemble), 88.76% (zero-shot), and 75.02% (keyword filter). Latency is compared (47ms vs 598ms). The magnitude of differences is clear."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for the dataset size. The paper uses ~28K samples (HarmBench-28K) but does not justify whether this is sufficient for the claims made. Also, Section III.C.2 claims training on '56,000 samples from D1 and D2' which contradicts the earlier statement that D2 was reserved for evaluation only."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measure is reported for any result. No mention of multiple experimental runs. All results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table III compares against Static Keyword Filter, Zero-Shot Classifier, and Ensemble Moderation Pipeline. Table VI compares against JailbreakTracer, LLM-Sentry, and JBShield from the literature. However, Table VI explicitly notes results are 'from original publications and not from a unified dataset.'"
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Table VI includes JailbreakTracer (2025), LLM-Sentry (2024), and JBShield (2025), which are recent systems. However, the Table III baselines (Static Keyword Filter, generic Zero-Shot, Ensemble Filter) are generic and unspecified rather than named contemporary systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has four major components (SBERT-FAISS retrieval, fine-tuned classifier, zero-shot NLI, HITL feedback) but no ablation study is presented showing the individual contribution of each component. The paper claims HITL 'improved recall by 4.2%' but this is not from a controlled ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Tables III-VI report Accuracy, Precision, Recall, F1 Score, AUC, ASR (Attack Success Rate), FPR (False Positive Rate), and Average Latency."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system's detection outputs. The HITL component is described as part of the system architecture for labeling ambiguous cases, not as an evaluation methodology. All evaluation is fully automated against labeled datasets."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section III.B states the dataset was 'split into training (70%), validation (15%), and testing (15%) using stratified sampling.' Cross-dataset evaluation (Table V) uses entirely separate benchmarks (JailbreakV-28K, JBB-Behaviors, JailbreakTracer)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table IV provides per-language and per-LLM breakdown of detection performance across 5 languages and 4 models. Table V shows per-dataset results across 4 external benchmarks. Table II categorizes different attack types."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.D discusses the single false negative: 'a Unicode homoglyph variant.' Section IV.D (Performance Visualization) notes 'seven false positives emerged, all stemming from benign prompts with scientific or technical terminology (e.g., bomb calorimeter).'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results reported. Every experiment and configuration shows strong positive performance. No mention of approaches that were tried and failed, or configurations that degraded performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims '99.96% detection rate' but the confusion matrix analysis in Section IV.D states '99.996% detection rate.' The abstract claims 'F1 = 1.00' but Table III shows F1 = 99.98%. The paper also claims the system 'outperforms leading baselines such as LlamaGuard-2 (1.3%) and OpenAI Moderation (3.7%)' but Table VI acknowledges these comparisons are 'not from a unified dataset.' Numbers are internally inconsistent."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims ('HITL feedback further improved recall by 4.2% and lowered false positives by 11%', 'The hybrid architecture plays a central role') without ablation studies or controlled experiments to justify them. No component-level analysis isolates the causal contribution of individual modules."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the system works 'across over 100 languages' (abstract, Section III.B) but only tests 5 languages (English, French, Spanish, Arabic, Hindi). The title claims 'Multilingual' defense and the conclusion claims 'a new state-of-the-art in adversarial LLM defense' — vastly exceeding what was tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. The paper does not consider confounds such as: whether the dataset is too easy for any classifier, whether the near-perfect results indicate overfitting to the specific dataset distribution, or whether simpler approaches could achieve similar results on this particular benchmark."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures prompt classification accuracy on static datasets and frames it as 'real-time defense against adversarial LLM jailbreaks.' No discussion of the gap between classifying curated benchmark prompts and defending against adaptive real-world adversaries who modify their attacks in response to the defense system."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'distilbert-base-uncased' and 'facebook/bart-large-mnli' for its own components, but the LLMs tested against (GPT-4o, Claude 3 Opus, Gemini Flash, Mistral 7B) have no version/snapshot dates. Per schema, marketing names without snapshot dates do not count as specified versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The zero-shot classification module uses candidate labels '{harmful, safe}' for NLI entailment, but the exact input formatting and any prompt templates for the RAG comparator are not provided. The reader cannot reconstruct the exact inputs sent to each inference module."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.A reports: batch size 8, learning rate 2×10⁻⁵, 3 epochs, linear decay schedule, fixed token length 64. The SBERT embedding model and FAISS indexing parameters are mentioned but top-k retrieval count is not specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Sentra-Guard is a classification pipeline (embedding + classification + NLI + fusion), not an agentic system with tool use, memory, or iterative reasoning."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III.A-B describes: duplicate removal, filtering system-role instructions, discarding metadata, binary labeling (0/1), tokenization with distilbert-base-uncased, padding/truncation to 64 tokens, stratified 70/15/15 split. Neural machine translation for non-English prompts is also documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section exists. Section V ('Conclusion and Future Work') mentions future directions (generation-time monitoring, multimodal defenses) but does not discuss limitations of the current work. The discussion section (Section IV) analyzes performance but does not acknowledge weaknesses."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. No consideration of internal validity (e.g., dataset representativeness), external validity (e.g., generalization to real adversaries), or construct validity (e.g., whether benchmark detection equals real-world defense)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes unbounded claims: 'establishes a new state-of-the-art in adversarial LLM defense' (abstract/conclusion), 'over 100 languages' (tested only 5), 'enterprise-ready performance.' No explicit statements about what the results do NOT show or what settings were NOT tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The primary evaluation dataset D1 (HarmBench-28K) is described as an 'open-access adversarial benchmark.' Cross-dataset benchmarks (JBB-Behaviors, JailbreakTracer) are also public. However, the specific train/val/test splits used and any processed versions are not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "D1 is identified as HarmBench-28K with a description of its coverage areas (misinformation, cyberattacks, etc.). However, D2 is described only as 'a smaller set of adversarial prompts from publicly available red-teaming repositories' without naming the specific sources. Section III.C.2 claims 'trained on over 56,000 samples from D1 and D2' which contradicts Section III.A's statement that D2 was 'not included in training.'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from public adversarial prompt benchmarks (standard benchmark data sources)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The preprocessing steps are described (dedup, filter, label, split), but there are unexplained inconsistencies: D1 is 'HarmBench-28K' (~28K samples) yet Section III.C.2 claims training on '56,000 samples from D1 and D2' while D2 was stated to be excluded from training. The confusion matrix shows 24,145 harmful prompts in the test set, but 15% of 28K is only ~4,200. These numbers do not add up."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The Acknowledgement section thanks open-source communities and libraries but does not mention any funding source, grant, or financial support for the research."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "Author names are listed but institutional affiliations are not visible in the paper text. No affiliation information accompanies the author list in the provided content."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed. Appears to be unfunded academic work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests a custom defense tool (classifier pipeline), not a pre-trained model's capability on benchmarks. The system fine-tunes its own classifier on labeled data rather than evaluating pre-trained model knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Paper tests a defense tool rather than evaluating pre-trained model knowledge on benchmarks. Train/test splitting is handled through stratified sampling of D1, which is a standard practice for training custom classifiers."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Paper tests a defense tool rather than evaluating pre-trained model knowledge. Contamination in the schema's sense (model training data including benchmark) does not apply to a custom-trained classifier."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The HITL component is described architecturally but no human subjects study was conducted."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The ethics section (Section IV.E) discusses data ethics and responsible AI but does not involve human subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Average inference latency of 47ms is reported (Section IV.A, Table III). Per-language latency ranges from 42ms to 56ms (Table IV). Comparison with baseline latencies (385ms zero-shot, 598ms ensemble) is also provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.A states 'Training the classifier required about two hours' on a 'Tesla T4 GPU (8 GB), with auxiliary computations on an Apple M1 CPU.' This provides the hardware and training duration."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from a single training/evaluation run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results come from a single run or are averaged across multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No mention of hyperparameter search. The learning rate (2×10⁻⁵), batch size (8), and epochs (3) appear chosen without reported justification or search process."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the final configuration was selected. The paper presents one configuration with no discussion of alternatives tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes multiple comparisons across baselines (Tables III, VI), languages (Table IV), and datasets (Table V) without any correction for multiple comparisons. No statistical tests are performed at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system against baselines without acknowledging self-comparison bias. Table III baselines (Static Keyword Filter, Zero-Shot, Ensemble) appear to be the authors' own implementations, with no discussion of whether these faithfully represent the baseline approaches."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While latency is compared across systems (47ms vs 598ms), performance is not reported as a function of compute budget. The hybrid system likely uses substantially more parameters/compute than the keyword filter, but this is not controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HarmBench-28K actually measures real-world jailbreak defense capability. No analysis of the gap between detecting curated benchmark prompts and defending against adaptive adversaries in production."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Sentra-Guard is a classification pipeline, not an agentic system with scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The paper does not address whether the pre-trained models used (DistilBERT, SBERT, BART-MNLI) were trained on data that includes adversarial prompt examples similar to those in the test set."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The FAISS knowledge base contains both harmful and benign exemplars that could overlap structurally with test examples, but this is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train and test examples in HarmBench-28K share structural similarities, come from the same sources, or contain near-duplicates. The stratified split does not ensure independence."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is described beyond the basic train/val/test split."
       }
     }
   },
   "claims": [
     {
       "claim": "Sentra-Guard achieves 99.98% accuracy with AUC = 1.00, F1 = 1.00, and ASR of 0.004% on HarmBench-28K",
       "evidence": "Table III reports 99.98% accuracy, 100% precision, 99.97% recall, 99.98% F1. Figure 3 shows AUC = 1.00. Confusion matrix (Fig. 5) shows 1 false negative out of 24,145 harmful prompts. However, the abstract states '99.96% detection rate' and 'F1 = 1.00' which conflict with Table III (F1 = 99.98%) and the confusion matrix (99.996% detection rate).",
       "supported": "weak"
     },
     {
       "claim": "Sentra-Guard outperforms LlamaGuard-2 (1.3% ASR) and OpenAI Moderation (3.7% ASR)",
       "evidence": "Table VI and Table I compare ASR. However, Table VI explicitly notes: 'Results are drawn from original publications and not from a unified dataset.' The comparison is across different benchmarks, making it methodologically invalid as a direct comparison.",
       "supported": "weak"
     },
     {
       "claim": "The system achieves multilingual detection with >96% detection rate across 5 languages",
       "evidence": "Table IV shows detection rates ranging from 94.3% (Mistral 7B, Arabic) to 99.5% (Claude 3 Opus, English) across English, French, Spanish, Arabic, and Hindi on 4 LLM platforms.",
       "supported": "moderate"
     },
     {
       "claim": "HITL feedback improved recall by 4.2% and reduced false positives by 11% after 500 new prompts",
       "evidence": "Section IV.D states this claim but provides no detailed methodology — no before/after comparison table, no description of the 500 prompts, and no controlled experiment isolating HITL from other factors.",
       "supported": "weak"
     },
     {
       "claim": "Average inference latency of 47ms enables real-time deployment",
       "evidence": "Section IV.A and Table III report 47ms average latency. Table IV shows per-language latency from 42-56ms. Compared to 385ms (zero-shot) and 598ms (ensemble) baselines.",
       "supported": "moderate"
     },
     {
       "claim": "Cross-dataset generalization above 99.8% accuracy on JailbreakV-28K, JBB-Behaviors, and JailbreakTracer",
       "evidence": "Table V shows accuracy 98.88-99.96% across 4 external test sets. However, the cross-dataset evaluation methodology is not described in detail — it is unclear how the system was applied to these external datasets or whether any adaptation was performed.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Grossly misattributed references",
       "detail": "Multiple references in the paper do not match the cited content. [10] Woszczyk et al. is about 'dementia obfuscation in transcribed speech' but cited for HITL methods. [14] Li W. et al. is about 'phishing website detection' but cited for 'lexical obfuscation attacks using synthetic datasets.' [15] Liu et al. is about 'transformer models in advancing blockchain technology' but cited for 'ensemble transformer framework achieving 92% detection accuracy.' [16] Musial et al. is about 'earth observation retrieval' but cited for 'hybrid retrieval-classifier system with FAISS.' [17] Askari et al. is about 'few-shot image classification' but cited for 'fine-tuned transformers for semantic decoding.' [20] Durmus et al. is about 'subjective global opinions in language models' but cited for 'prompt classifiers + safety layers with 92.5% accuracy.' This pattern of systematic misattribution is a hallmark of AI-generated text and raises fundamental integrity concerns about the entire paper."
     },
     {
       "flag": "Results too good to be true",
       "detail": "AUC = 1.00 (perfect), F1 ≈ 1.00, and 99.996% detection rate on a complex adversarial classification task with 28K+ samples is extraordinarily suspicious. Only 1 false negative and 7 false positives would be remarkable even for trivial classification tasks. The near-perfect metrics across all evaluation dimensions suggest potential issues with the evaluation methodology, dataset difficulty, or data integrity."
     },
     {
       "flag": "Internally inconsistent numbers",
       "detail": "The abstract states '99.96% detection rate' but the confusion matrix yields 99.996%. The abstract says 'F1 = 1.00' but Table III shows F1 = 99.98%. Table III lists False Positives as 'Low (~83%)' which is nonsensical — 83% false positives is not 'Low,' and the body text states FPR is 0.03%. The text states '99.97recall' (missing percentage sign and formatting). These inconsistencies suggest the paper was not carefully reviewed by the authors."
     },
     {
       "flag": "Contradictory dataset claims",
       "detail": "Section III.A states D2 was 'not included in training; instead, they were reserved for inference-time evaluation.' Yet Section III.C.2 states 'The classifier has been trained on over 56,000 samples from D1 and D2.' D1 is HarmBench-28K (~28K samples), so 56K would require D2 in training. Additionally, the confusion matrix shows 24,145 harmful test prompts, but 15% of 28K is only ~4,200 — the numbers do not reconcile."
     },
     {
       "flag": "Unfair cross-study comparison",
       "detail": "Table VI compares Sentra-Guard against JailbreakTracer, LLM-Sentry, and JBShield, but explicitly acknowledges 'Results are drawn from original publications and not from a unified dataset.' This makes the ASR comparison (0.004% vs 8.1%, 10%, <59%) methodologically invalid. Different datasets have different difficulty levels."
     },
     {
       "flag": "No ablation study despite multi-component system",
       "detail": "The system combines 4 major components (SBERT-FAISS retrieval, fine-tuned classifier, zero-shot NLI, HITL feedback) but presents no ablation study. It is impossible to determine which components contribute to performance and whether all are necessary. The claimed HITL improvement (4.2% recall, 11% FP reduction) is stated without controlled experimental support."
     },
     {
       "flag": "Fabricated benchmark name",
       "detail": "The paper references 'HarmBench-28K' as its primary dataset (D1), and 'HarmBench v2.3' (attributed to [28] Yan et al., who actually wrote about adversarial metaphors). The real HarmBench (Mazeika et al., 2024) contains ~510 behaviors, not 28K. 'HarmBench-28K' does not appear to be a real benchmark, or it is a renamed/modified version without proper attribution."
     },
     {
       "flag": "No statistical rigor whatsoever",
       "detail": "The paper reports no error bars, no confidence intervals, no significance tests, no multiple-run analysis, no seed sensitivity. For a paper claiming 'new state-of-the-art' with near-perfect metrics, the complete absence of statistical rigor makes the results unverifiable."
     }
   ],
   "cited_papers": [
     {
       "title": "Privacy in large language models: Attacks, defenses and future directions",
       "authors": ["H. Li", "Y. Chen", "J. Luo"],
       "year": 2023,
       "arxiv_id": "2310.10383",
       "relevance": "Survey of privacy attacks and defenses for LLMs, directly relevant to LLM safety research."
     },
     {
       "title": "Jailbreaking and mitigation of vulnerabilities in large language models",
       "authors": ["B. Peng", "K. Chen", "Q. Niu"],
       "year": 2024,
       "arxiv_id": "2410.15236",
       "relevance": "Comprehensive study of jailbreak vulnerabilities and mitigations in LLMs."
     },
     {
       "title": "Attack and defense techniques in large language models: A survey and new perspectives",
       "authors": ["Z. Liao", "K. Chen", "Y. Lin"],
       "year": 2025,
       "arxiv_id": "2505.00976",
       "relevance": "Survey of attack and defense techniques for LLMs, covering the adversarial threat landscape."
     },
     {
       "title": "JailGuard: A universal detection framework for prompt-based attacks on LLM systems",
       "authors": ["X. Zhang", "C. Zhang", "T. Li"],
       "year": 2025,
       "relevance": "Universal jailbreak detection framework for LLMs, directly comparable defense system."
     },
     {
       "title": "AutoDefense: Multi-agent LLM defense against jailbreak attacks",
       "authors": ["Y. Zeng", "Y. Wu", "X. Zhang"],
       "year": 2024,
       "arxiv_id": "2403.04783",
       "relevance": "Multi-agent approach to LLM jailbreak defense using chain-of-thought and LlamaGuard."
     },
     {
       "title": "Llama Guard: LLM-based input-output safeguard for human-AI conversations",
       "authors": ["H. Inan", "K. Upasani", "J. Chi"],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "Foundational LLM safety guardrail system from Meta, used as baseline in LLM defense evaluations."
     },
     {
       "title": "Do anything now: Characterizing and evaluating in-the-wild jailbreak prompts on large language models",
       "authors": ["X. Shen", "Z. Chen", "M. Backes"],
       "year": 2024,
       "relevance": "Empirical characterization of real-world jailbreak prompts targeting LLMs, including the DAN taxonomy."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["L. Ouyang", "J. Wu", "X. Jiang"],
       "year": 2022,
       "relevance": "Foundational RLHF paper for instruction-following LLMs, establishes the alignment approach that jailbreaks attempt to bypass."
     },
     {
       "title": "A cross-language investigation into jailbreak attacks in large language models",
       "authors": ["J. Li", "Y. Liu", "C. Liu"],
       "year": 2024,
       "arxiv_id": "2401.16765",
       "relevance": "Cross-lingual jailbreak attack analysis demonstrating multilingual vulnerability of LLMs."
     },
     {
       "title": "JailbreakTracer: Explainable detection of jailbreaking prompts in LLMs using synthetic data generation",
       "authors": ["M. F. A. Sayeedi", "M. B. Hossain"],
       "year": 2025,
       "relevance": "Explainable jailbreak detection system using synthetic data, directly comparable defense approach."
     },
     {
       "title": "LLM-Sentry: A model-agnostic human-in-the-loop framework for securing large language models",
       "authors": ["S. Irtiza", "K. A. Akbar", "A. Yasmeen"],
       "year": 2024,
       "relevance": "Model-agnostic HITL defense framework for LLMs, directly comparable to Sentra-Guard's approach."
     },
     {
       "title": "JBShield: Defending large language models from jailbreak attacks through activated concept analysis and manipulation",
       "authors": ["S. Zhang", "Y. Zhai", "K. Guo"],
       "year": 2025,
       "arxiv_id": "2502.07557",
       "relevance": "Concept-based jailbreak defense for LLMs using activation analysis."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Jailbreak detection is a real deployment need, but no code is released and the system's integrity is questionable given the reference fabrication issues."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected pattern that hybrid classifiers can detect adversarial prompts; no contrarian findings."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Addresses AI safety concerns by demonstrating jailbreak attacks across major LLMs (GPT-4o, Claude, Gemini, Mistral) and the Table II attack examples are attention-grabbing."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenge to established players; straightforward system proposal."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code released, no demo, no pip-installable tool. Only a promise of future release."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Unknown authors with no visible institutional affiliation or major lab connection."
     }
   }
 }