ai-research-survey

scan.json (27409B)

---

 {
   "paper": {
     "title": "Method of Counteracting Manipulative Queries to Large Language Models",
     "authors": ["Yehor Kovalchuk", "Mykhailo Kolomytsev"],
     "year": 2025,
     "venue": "Intelligent Data Analysis Methods in Cybersecurity (KPI)",
     "doi": "10.20535/tacs.2664-29132025.3.345389"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes a Multi-Head DistilBERT classifier with four specialized detection heads (malicious intent, instruction override, persona adoption, high-risk action) for detecting prompt injection and jailbreaking attacks. Using a hybrid data generation strategy with GPT-5 as a teacher model for knowledge distillation, the system achieves 0.99 Recall on a real-world holdout set, significantly outperforming keyword matching (0.80) and TF-IDF + Logistic Regression (0.95) baselines. The DistilBERT model maintains F1 of 0.98 on obfuscated prompts where keyword-based filters collapse to 0.27.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or supplementary materials link is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper references public datasets (deepset/prompt-injections, jackhhao/jailbreak-classification) but does not release its own 3,000 synthetic samples or the GPT-5 teacher-labeled data, which are the primary dataset contributions."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or library versions are provided. The paper mentions DistilBERT and PyTorch (via references) but gives no version information or environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The architectural description in Section 1 is conceptual only."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (F1, Recall, Precision) are reported as point estimates with no confidence intervals or error bars. Figures 3 and 4 show bar charts without uncertainty."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the proposed system 'significantly outperforming' baselines but no statistical significance tests (t-tests, bootstrap, etc.) are performed. The claim is based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute performance numbers with baseline context are provided: keyword F1 drops from ~0.95 (Easy) to 0.27 (Medium), while proposed method maintains 0.98. Recall: keyword 0.80, TF-IDF 0.95, proposed 0.99. Sufficient context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 3,000 synthetic samples is stated without justification. No power analysis or rationale for why this sample size is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be from single experimental runs with no information about stability across seeds or trials."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines are included: Keyword Matching (RegEx with 200+ blacklisted terms) and TF-IDF + Logistic Regression, described in Section 2.1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "Keyword matching and TF-IDF + Logistic Regression are not contemporary baselines for prompt injection detection in 2025. More relevant baselines would include other fine-tuned transformer classifiers, dedicated prompt injection detectors (e.g., Lakera Guard, Rebuff), or at minimum a standard BERT classifier."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (4 detection heads, Paranoid Mode ensemble logic, knowledge distillation) but no ablation study is performed to measure the contribution of individual heads or the Paranoid Mode."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: Accuracy, F1-Score, Recall, and Precision (Sections 2.2 and 2.3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed. All evaluation is automated against labeled datasets. No manual inspection of whether blocked prompts are correctly classified."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "A 20% holdout set from the deepset/prompt-injections dataset is used for real-world evaluation, separate from the synthetic training data (Section 2.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset difficulty: Synthetic Easy, Synthetic Medium, Synthetic Hard, and Real-World Holdout (Figures 3 and 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or error analysis is provided. The paper does not discuss what types of attacks the system misses or what causes false positives."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. Every comparison shows the proposed method outperforming baselines. The slight precision reduction (0.89 vs 0.94) is framed as a 'deliberate trade-off' rather than a negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims Recall of 0.99 and outperformance of TF-IDF and keyword baselines, both supported by Figures 3 and 4 in the results section."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes the improved detection to the multi-head architecture and knowledge distillation ('the proposed Multi-Head architecture successfully resolved these ambiguities') but provides no ablation to isolate these contributions. A single-head DistilBERT baseline trained on the same data would be needed to justify these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims suitability for 'real-time protection of SIEM and SOAR systems' and 'deployment on edge devices' but tests only on synthetic data and one public NLP dataset. No evaluation in actual SIEM/SOAR contexts or on edge hardware is provided. The title claims generality ('Counteracting Manipulative Queries to Large Language Models') beyond what was tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered. The paper does not discuss whether DistilBERT's sub-word tokenization alone (without multi-head design) could explain the obfuscation robustness, or whether the synthetic training data simply matches the test distribution."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures classification metrics on labeled datasets but claims real-world protection effectiveness for SIEM/SOAR systems. The gap between academic dataset performance and deployed system effectiveness in adversarial environments is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'DistilBERT' and 'GPT-5' without version identifiers, checkpoint dates, or specific model IDs. No API version or snapshot date for GPT-5."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'a custom system prompt containing the definitions of specific attack patterns' used for GPT-5 teacher labeling but does not provide the actual prompt text. Only natural language descriptions are given."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only λ=0.5 (loss weighting) and 0.8 (Paranoid Mode threshold) are reported. Critical training hyperparameters (learning rate, batch size, epochs, optimizer, warm-up schedule) are missing."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a classification middleware, not an agentic workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The synthetic data generation process is described conceptually (three difficulty levels, obfuscation techniques) but specific preprocessing of the public datasets (filtering, deduplication, class balancing, train/test splits beyond the 20% holdout) is not documented. No counts at each pipeline stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The Conclusions section contains no limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not state what attack types, languages, or deployment contexts are excluded from its claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is made available. The 3,000 synthetic samples and GPT-5 teacher labels are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.2 describes the data collection: synthetic generation across three difficulty levels with specific obfuscation techniques (token splitting, leetspeak, noise injection), and use of named public datasets (deepset/prompt-injections, jackhhao/jailbreak-classification) processed through GPT-5 teacher."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public datasets plus synthetic generation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline is described at a high level (generate → label → train) with a figure (Figure 2), but specific counts at each stage, filtering criteria, and how the final training set was composed from the different sources are not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: National Technical University of Ukraine 'Igor Sikorsky Kyiv Polytechnic Institute', Institute of Physics and Technology."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding disclosed. Appears to be unfunded university research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests a defense system (prompt injection classifier), not a pre-trained model's capability on a benchmark. Contamination of pre-training data is not the relevant concern."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper evaluates a trained classifier, not a pre-trained model's knowledge."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — contamination in the traditional sense (benchmark in pre-training data) is not applicable to this defense evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'low computational latency suitable for deployment on edge devices' but provides no actual latency measurements, throughput numbers, or inference cost data."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training time, GPU hours, or computational budget is reported. The cost of GPT-5 teacher labeling (API calls for 3,000+ samples) is also unstated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or results across multiple seeds. All results appear to be from a single run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented as single values without any indication of repetition."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "λ=0.5 is described as 'empirically selected' and threshold 0.8 from 'ROC curve analysis' but no search budget, number of configurations tried, or search methodology is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The selection of λ=0.5 and threshold 0.8 is mentioned but the selection process is not described beyond 'empirically selected' and 'ROC curve analysis of the validation set.' No details on how many configurations were evaluated."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "Only a few pairwise comparisons (3 methods × 4 datasets) are made, and no statistical tests are performed in the first place."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implemented all baselines and their own system without acknowledging the bias of evaluating their own implementation. No independent evaluation or discussion of this bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison of computational cost between methods. DistilBERT is far more expensive than keyword matching or TF-IDF, but this compute difference is not discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the synthetic datasets or deepset/prompt-injections holdout actually represent real-world prompt injection attacks in SIEM/SOAR contexts. The gap between academic datasets and production threat landscape is not addressed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved; this is a standalone classifier evaluation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether DistilBERT's pre-training data could contain examples from the deepset/prompt-injections test set, or whether the synthetic training data temporally overlaps with test patterns."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the synthetic data generation process introduces patterns that leak into the test set (e.g., if synthetic and real-world attacks share systematic features due to the generation process)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and test examples are independent. The GPT-5 teacher labels both synthetic training data and the real-world datasets were not checked for overlap or near-duplicates."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied. No deduplication, n-gram overlap analysis, or temporal splitting is mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "The proposed Multi-Head DistilBERT system achieves a Recall of 0.99 on real-world attack datasets.",
       "evidence": "Figure 4 shows Recall comparison on real-world holdout: proposed method 0.99, TF-IDF 0.95, keyword 0.80. Section 2.3 discusses results.",
       "supported": "moderate"
     },
     {
       "claim": "The proposed system maintains F1 of 0.98 on obfuscated prompts where keyword baselines collapse to 0.27.",
       "evidence": "Figure 3 shows F1-Scores across datasets. On Synthetic Medium (obfuscation), keyword baseline achieves 0.27 while proposed method achieves 0.98.",
       "supported": "moderate"
     },
     {
       "claim": "The multi-head architecture successfully resolves semantic ambiguities that defeat statistical approaches.",
       "evidence": "On Synthetic Hard dataset, TF-IDF achieves F1 of 0.60 while proposed method achieves 0.87 (Section 2.3). However, no ablation isolates the multi-head design as the cause.",
       "supported": "weak"
     },
     {
       "claim": "The solution operates with low computational latency suitable for deployment on edge devices.",
       "evidence": "Stated in the abstract and conclusions but no latency measurements, throughput benchmarks, or edge device testing is provided.",
       "supported": "unsupported"
     },
     {
       "claim": "Knowledge distillation from GPT-5 enables the compact student model to learn complex attack patterns.",
       "evidence": "The knowledge distillation pipeline is described in Section 1.2, but no comparison between models trained with and without teacher labels is provided to demonstrate the contribution of distillation.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Weak and outdated baselines",
       "detail": "Keyword matching and TF-IDF + Logistic Regression are not competitive baselines for prompt injection detection in 2025. More relevant comparisons would include fine-tuned BERT/RoBERTa classifiers or dedicated prompt injection detection tools, making the claimed improvements appear larger than they may be against state-of-the-art."
     },
     {
       "flag": "No error bars or multiple runs",
       "detail": "All results are single-point estimates with no confidence intervals, standard deviations, or indication of result stability. Given the small dataset (3,000 synthetic samples + holdout), variance across random seeds could significantly affect reported numbers."
     },
     {
       "flag": "Claims significantly outrun evidence",
       "detail": "The paper claims suitability for SIEM/SOAR real-time protection and edge device deployment without providing any latency measurements, throughput tests, or deployment evaluations. These are engineering claims with zero supporting data."
     },
     {
       "flag": "Self-evaluated synthetic data",
       "detail": "The majority of evaluation is on synthetic data the authors themselves generated. The generator's design may inadvertently create patterns that the model is specifically trained to detect, inflating apparent performance."
     },
     {
       "flag": "No ablation study",
       "detail": "The multi-head architecture, Paranoid Mode, and knowledge distillation are all claimed as contributions, but no ablation study isolates the effect of any individual component. A single-head DistilBERT with the same training data would be the minimum necessary control."
     },
     {
       "flag": "No limitations discussion",
       "detail": "The paper contains no limitations section, no threats to validity, and no scope boundaries. For a security system claiming production readiness, this is a significant omission — there is no discussion of adversarial robustness to adaptive attackers who know the defense exists."
     }
   ],
   "cited_papers": [
     {
       "title": "Universal and Transferable Adversarial Attacks on Aligned Language Models",
       "authors": ["A. Zou", "Z. Wang", "J. Z. Kolter", "M. Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Foundational work on adversarial attacks against LLM safety alignment, directly relevant to the attack vectors this paper aims to defend against."
     },
     {
       "title": "Not what you've signed up for: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection",
       "authors": ["K. Greshake"],
       "year": 2023,
       "relevance": "Demonstrates indirect prompt injection attacks on RAG systems, the threat model this paper's defense targets."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI Feedback",
       "authors": ["Y. Bai"],
       "year": 2022,
       "arxiv_id": "2212.08073",
       "relevance": "Proposes an alternative AI safety alignment approach (RLHF alternative) that this paper positions as insufficient against prompt injection."
     },
     {
       "title": "Jailbroken: How Does LLM Safety Training Fail?",
       "authors": ["A. Wei", "N. Haghtalab", "J. Steinhardt"],
       "year": 2024,
       "relevance": "Analyzes failure modes of LLM safety training including jailbreaking techniques, directly relevant to the attack taxonomy in this paper."
     },
     {
       "title": "DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter",
       "authors": ["V. Sanh", "L. Debut", "J. Chaumond", "T. Wolf"],
       "year": 2019,
       "arxiv_id": "1910.01108",
       "relevance": "The base model architecture used in this paper's Multi-Head classifier for prompt injection detection."
     },
     {
       "title": "Extracting Training Data from Large Language Models",
       "authors": ["N. Carlini"],
       "year": 2021,
       "relevance": "Demonstrates LLM security vulnerabilities through training data extraction, part of the broader LLM security landscape this paper addresses."
     },
     {
       "title": "Prompt Injection attack against LLM-integrated Applications",
       "authors": ["Y. Liu", "G. Deng", "Z. Xu"],
       "year": 2023,
       "arxiv_id": "2306.05499",
       "relevance": "Directly studies prompt injection attacks against LLM-integrated applications, the core threat this paper's method aims to counteract."
     },
     {
       "title": "Distilling the Knowledge in a Neural Network",
       "authors": ["G. Hinton", "O. Vinyals", "J. Dean"],
       "year": 2015,
       "arxiv_id": "1503.02531",
       "relevance": "Foundational knowledge distillation technique used in this paper's hybrid data labeling strategy (GPT-5 teacher → DistilBERT student)."
     },
     {
       "title": "OWASP Top 10 for Large Language Model Applications",
       "authors": ["OWASP Foundation"],
       "year": 2023,
       "relevance": "Industry standard LLM security framework that categorizes prompt injection as a top vulnerability, providing the threat taxonomy context for this work."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Middleware for prompt injection detection is practically useful for teams deploying LLMs in security contexts, but no code or tool is released."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected finding that semantic analysis outperforms keyword filtering — no surprising or contrarian claims."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Addresses prompt injection and jailbreaking in critical infrastructure (SIEM/SOAR), a real and growing concern in LLM security."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict — straightforward defense method proposal."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code release, no demo, no pip-installable tool — cannot be tried."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From a Ukrainian university lab with no brand recognition in the AI/ML community."
     }
   }
 }