scan.json (17004B)
1 { 2 "paper": { 3 "title": "Real Time AI Defense Against Prompt Injection Attacks", 4 "authors": [ 5 "Zarmeen Binte Nasir", 6 "Seher Salahuddin", 7 "Sumayya Shahid" 8 ], 9 "year": 2025, 10 "venue": "2nd International Conference on Cybersecurity and Digital Defense (ICyDD) 2025", 11 "doi": "10.37962/icydd/2025/23-24" 12 }, 13 "scan_version": 3, 14 "active_modules": [], 15 "methodology_tags": ["theoretical"], 16 "key_findings": "This paper is an abstract-only conference submission that proposes a layered, real-time defense methodology against prompt injection attacks on LLMs, integrating continuous monitoring and semantic deviation detection. No implementation, evaluation, or empirical results are presented. The entire paper consists of a single-page abstract and three references, with no methodology section, no experimental design, and no results.", 17 "checklist": { 18 "artifacts": { 19 "code_released": { 20 "applies": true, 21 "answer": false, 22 "justification": "No code, repository URL, or archive is provided anywhere in the paper. The paper proposes a 'modular API' but releases nothing." 23 }, 24 "data_released": { 25 "applies": true, 26 "answer": false, 27 "justification": "No dataset is released or referenced. The paper contains no experimental data of any kind." 28 }, 29 "environment_specified": { 30 "applies": true, 31 "answer": false, 32 "justification": "No environment specifications, dependencies, or technical requirements are mentioned. The paper is purely a proposal with no implementation details." 33 }, 34 "reproduction_instructions": { 35 "applies": true, 36 "answer": false, 37 "justification": "No reproduction instructions are provided. There is nothing to reproduce — the paper contains only an abstract describing a proposed approach." 38 } 39 }, 40 "statistical_methodology": { 41 "confidence_intervals_or_error_bars": { 42 "applies": false, 43 "answer": false, 44 "justification": "No experiments were conducted. The paper is a theoretical proposal with no quantitative results." 45 }, 46 "significance_tests": { 47 "applies": false, 48 "answer": false, 49 "justification": "No experiments were conducted. No comparative claims are backed by data." 50 }, 51 "effect_sizes_reported": { 52 "applies": false, 53 "answer": false, 54 "justification": "No experiments were conducted. No effect sizes could be reported." 55 }, 56 "sample_size_justified": { 57 "applies": false, 58 "answer": false, 59 "justification": "No experiments were conducted. There is no sample to justify." 60 }, 61 "variance_reported": { 62 "applies": false, 63 "answer": false, 64 "justification": "No experiments were conducted. No experimental runs were performed." 65 } 66 }, 67 "evaluation_design": { 68 "baselines_included": { 69 "applies": false, 70 "answer": false, 71 "justification": "No evaluation was conducted. The paper mentions existing defenses (input filtering, rule-based validation, output sanitization) in the abstract but performs no comparative evaluation." 72 }, 73 "baselines_contemporary": { 74 "applies": false, 75 "answer": false, 76 "justification": "No evaluation was conducted, so no baselines were tested." 77 }, 78 "ablation_study": { 79 "applies": false, 80 "answer": false, 81 "justification": "No system was implemented or evaluated, so no ablation study could be conducted." 82 }, 83 "multiple_metrics": { 84 "applies": false, 85 "answer": false, 86 "justification": "No evaluation was conducted. No metrics of any kind are reported." 87 }, 88 "human_evaluation": { 89 "applies": false, 90 "answer": false, 91 "justification": "No evaluation of any kind was conducted." 92 }, 93 "held_out_test_set": { 94 "applies": false, 95 "answer": false, 96 "justification": "No evaluation was conducted. No datasets were used." 97 }, 98 "per_category_breakdown": { 99 "applies": false, 100 "answer": false, 101 "justification": "No evaluation was conducted. No results exist to break down." 102 }, 103 "failure_cases_discussed": { 104 "applies": false, 105 "answer": false, 106 "justification": "No system was implemented or tested, so no failure cases could be observed or discussed." 107 }, 108 "negative_results_reported": { 109 "applies": false, 110 "answer": false, 111 "justification": "No experiments were conducted. No results of any kind are reported." 112 } 113 }, 114 "claims_and_evidence": { 115 "abstract_claims_supported": { 116 "applies": true, 117 "answer": false, 118 "justification": "The abstract claims the proposed approach offers 'continuous self-learning security, enhanced model reliability, minimized false positives and improved user trust.' None of these claims are supported by any evidence in the paper — there is no results section, no evaluation, and no implementation." 119 }, 120 "causal_claims_justified": { 121 "applies": true, 122 "answer": false, 123 "justification": "The paper makes causal claims: 'By implementing this adaptive methodology, the proposed approach offers several benefits including continuous self-learning security, enhanced model reliability, minimized false positives.' No study design or evidence supports these causal statements." 124 }, 125 "generalization_bounded": { 126 "applies": true, 127 "answer": false, 128 "justification": "The paper claims the approach can be 'easily integrated into SaaS and enterprise applications while ensuring scalability and efficiency' and aims to 'build more robust, secure and resilient foundation for LLM based applications and AI deployments.' These broad generalizations are unbounded and unsupported by any evidence." 129 }, 130 "alternative_explanations_discussed": { 131 "applies": false, 132 "answer": false, 133 "justification": "The paper presents no empirical results, so alternative explanations for results are not applicable." 134 }, 135 "proxy_outcome_distinction": { 136 "applies": false, 137 "answer": false, 138 "justification": "The paper is a theoretical proposal with no measurements. No proxy-outcome distinction is applicable." 139 } 140 }, 141 "setup_transparency": { 142 "model_versions_specified": { 143 "applies": false, 144 "answer": false, 145 "justification": "No models were used in any experiment. The paper is a theoretical proposal without implementation." 146 }, 147 "prompts_provided": { 148 "applies": false, 149 "answer": false, 150 "justification": "No prompting was done. The paper is a theoretical proposal." 151 }, 152 "hyperparameters_reported": { 153 "applies": false, 154 "answer": false, 155 "justification": "No experiments were conducted. No hyperparameters to report." 156 }, 157 "scaffolding_described": { 158 "applies": false, 159 "answer": false, 160 "justification": "No system was implemented. The paper is a theoretical proposal." 161 }, 162 "data_preprocessing_documented": { 163 "applies": false, 164 "answer": false, 165 "justification": "No data was collected or processed. The paper is a theoretical proposal." 166 } 167 }, 168 "limitations_and_scope": { 169 "limitations_section_present": { 170 "applies": true, 171 "answer": false, 172 "justification": "There is no limitations section. The paper consists only of an abstract and references with no body text discussing limitations." 173 }, 174 "threats_to_validity_specific": { 175 "applies": true, 176 "answer": false, 177 "justification": "No threats to validity are discussed. The paper has no body text beyond the abstract." 178 }, 179 "scope_boundaries_stated": { 180 "applies": true, 181 "answer": false, 182 "justification": "No scope boundaries are stated. The abstract makes broad claims about applicability to 'SaaS and enterprise applications' without any qualification of scope." 183 } 184 }, 185 "data_integrity": { 186 "raw_data_available": { 187 "applies": false, 188 "answer": false, 189 "justification": "No data was collected. The paper is a theoretical proposal with no experiments." 190 }, 191 "data_collection_described": { 192 "applies": false, 193 "answer": false, 194 "justification": "No data was collected. The paper is a theoretical proposal." 195 }, 196 "recruitment_methods_described": { 197 "applies": false, 198 "answer": false, 199 "justification": "No participants were recruited and no data sources were used. The paper is a theoretical proposal." 200 }, 201 "data_pipeline_documented": { 202 "applies": false, 203 "answer": false, 204 "justification": "No data pipeline exists. The paper is a theoretical proposal with no experiments." 205 } 206 }, 207 "conflicts_of_interest": { 208 "funding_disclosed": { 209 "applies": true, 210 "answer": false, 211 "justification": "No funding source is disclosed anywhere in the paper. There is no acknowledgments section." 212 }, 213 "affiliations_disclosed": { 214 "applies": true, 215 "answer": true, 216 "justification": "Author affiliations are clearly listed: Department of Computer Science and Software Engineering, Jinnah University for Women, Karachi, Pakistan." 217 }, 218 "funder_independent_of_outcome": { 219 "applies": false, 220 "answer": false, 221 "justification": "No funding is disclosed. This appears to be unfunded university student work." 222 }, 223 "financial_interests_declared": { 224 "applies": true, 225 "answer": false, 226 "justification": "No competing interests or financial interests statement is present in the paper." 227 } 228 }, 229 "contamination": { 230 "training_cutoff_stated": { 231 "applies": false, 232 "answer": false, 233 "justification": "The paper does not evaluate any pre-trained model on any benchmark. It is a theoretical proposal." 234 }, 235 "train_test_overlap_discussed": { 236 "applies": false, 237 "answer": false, 238 "justification": "The paper does not evaluate any pre-trained model on any benchmark." 239 }, 240 "benchmark_contamination_addressed": { 241 "applies": false, 242 "answer": false, 243 "justification": "The paper does not evaluate any pre-trained model on any benchmark." 244 } 245 }, 246 "human_studies": { 247 "pre_registered": { 248 "applies": false, 249 "answer": false, 250 "justification": "No human participants are involved in this paper." 251 }, 252 "irb_or_ethics_approval": { 253 "applies": false, 254 "answer": false, 255 "justification": "No human participants are involved in this paper." 256 }, 257 "demographics_reported": { 258 "applies": false, 259 "answer": false, 260 "justification": "No human participants are involved in this paper." 261 }, 262 "inclusion_exclusion_criteria": { 263 "applies": false, 264 "answer": false, 265 "justification": "No human participants are involved in this paper." 266 }, 267 "randomization_described": { 268 "applies": false, 269 "answer": false, 270 "justification": "No human participants are involved in this paper." 271 }, 272 "blinding_described": { 273 "applies": false, 274 "answer": false, 275 "justification": "No human participants are involved in this paper." 276 }, 277 "attrition_reported": { 278 "applies": false, 279 "answer": false, 280 "justification": "No human participants are involved in this paper." 281 } 282 }, 283 "cost_and_practicality": { 284 "inference_cost_reported": { 285 "applies": false, 286 "answer": false, 287 "justification": "The paper is a theoretical proposal with no implementation or experiments. Cost is not applicable." 288 }, 289 "compute_budget_stated": { 290 "applies": false, 291 "answer": false, 292 "justification": "The paper is a theoretical proposal with no implementation or experiments." 293 } 294 } 295 }, 296 "claims": [ 297 { 298 "claim": "Existing defense methods (input filtering, rule-based validation, output sanitization) fail against evolving and semantically adaptive attacks.", 299 "evidence": "Stated in the abstract as motivation but no empirical evidence, comparison, or citations to support this specific claim.", 300 "supported": "unsupported" 301 }, 302 { 303 "claim": "The proposed layered methodology integrates continuous monitoring and identification of semantic deviations to provide real-time defense against prompt injection.", 304 "evidence": "Described only at a high level in the abstract. No implementation details, architecture diagrams, algorithms, or technical specifications are provided.", 305 "supported": "unsupported" 306 }, 307 { 308 "claim": "The system instantly blocks and sanitizes malicious prompts while preserving natural responses' fluency and accuracy.", 309 "evidence": "No evidence provided. No evaluation of blocking effectiveness, false positive rates, or impact on response quality.", 310 "supported": "unsupported" 311 }, 312 { 313 "claim": "The approach offers continuous self-learning security, enhanced model reliability, minimized false positives and improved user trust.", 314 "evidence": "No evidence provided. These are aspirational claims with no supporting experiments or data.", 315 "supported": "unsupported" 316 } 317 ], 318 "red_flags": [ 319 { 320 "flag": "Abstract-only paper with no content", 321 "detail": "The entire paper consists of a single-page abstract and three references. There is no methodology section, no implementation, no evaluation, no results, and no discussion. This is a conference abstract, not a research paper." 322 }, 323 { 324 "flag": "Claims massively outrun evidence", 325 "detail": "The paper makes sweeping claims about 'continuous self-learning security,' 'minimized false positives,' 'scalability and efficiency,' and suitability for 'SaaS and enterprise applications' without a single experiment, prototype, or data point." 326 }, 327 { 328 "flag": "No technical depth", 329 "detail": "The proposed 'layered methodology' is described only in vague terms ('continuous monitoring,' 'identification of semantic deviations,' 'blocks and sanitizes malicious prompts'). No algorithms, architectures, or technical mechanisms are specified." 330 }, 331 { 332 "flag": "Extremely sparse references", 333 "detail": "Only 3 references are cited, all from 2025. No foundational work on prompt injection (e.g., Perez & Ribeiro, Greshake et al.), LLM security, or anomaly detection is referenced." 334 } 335 ], 336 "cited_papers": [ 337 { 338 "title": "LLM adversarial prompt attack detection and mitigation engine: A novel framework for securing generative AI systems", 339 "authors": ["M. Fathima"], 340 "year": 2025, 341 "relevance": "Directly addresses LLM prompt attack detection and mitigation, core to the survey's AI security scope." 342 }, 343 { 344 "title": "Multimodal LLM-guided sequential detection of cyber threats in electric vehicle charging systems", 345 "authors": ["R. Honnalli", "J. Farooq"], 346 "year": 2025, 347 "relevance": "Explores LLM-guided cyber threat detection in critical infrastructure, relevant to agentic AI security applications." 348 }, 349 { 350 "title": "Cyberattacks on large language models: Attack detection and architecture adaptability", 351 "authors": ["S. Alla", "A. S. Sichani"], 352 "year": 2025, 353 "relevance": "Addresses LLM attack detection and architectural defenses, directly relevant to AI security evaluation." 354 } 355 ], 356 "engagement_factors": { 357 "practical_relevance": { 358 "score": 0, 359 "justification": "No working tool, code, or usable technique is provided — the paper is purely a high-level proposal." 360 }, 361 "surprise_contrarian": { 362 "score": 0, 363 "justification": "Does not challenge any conventional wisdom; proposes a defense approach consistent with existing ideas in the space." 364 }, 365 "fear_safety": { 366 "score": 1, 367 "justification": "Discusses prompt injection attacks on LLMs, a relevant security concern, but adds no novel attack demonstrations or findings." 368 }, 369 "drama_conflict": { 370 "score": 0, 371 "justification": "No controversy, no critique of specific systems or organizations." 372 }, 373 "demo_ability": { 374 "score": 0, 375 "justification": "No code, no demo, no prototype — nothing for anyone to try." 376 }, 377 "brand_recognition": { 378 "score": 0, 379 "justification": "From Jinnah University for Women, published at a small regional conference (ICyDD). No brand recognition." 380 } 381 } 382 }