ai-research-survey

scan.json (27886B)

---

 {
   "paper": {
     "title": "Prompt Injection Detection in LLM Integrated Applications",
     "authors": [
       "Qianlong Lan",
       "Anuj Kaul",
       "Shaun Jones"
     ],
     "year": 2025,
     "venue": "International Journal of Network Dynamics and Intelligence",
     "doi": "10.53941/ijndi.2025.100013"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "The paper proposes a multi-faceted prompt injection detection system combining banned terms, vector embeddings, and a fine-tuned BERT model. Evaluation on ~250k validation records across 4 use cases shows BERT achieves 0.91-0.99 accuracy, with the general use case at 0.99 but the AI assistant use case at only 0.91. However, only the BERT component is evaluated; the banned terms and vector embedding components are described conceptually without empirical validation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Reference [56] links to a GitHub repo (lanqianlong/IJNDI) but this is cited as the source for manual annotation data, not the detection system code. No repository URL for the detection system itself is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses open-source datasets [44-55] which are publicly available, but the internal eBay product use case data is not released. The Data Availability Statement says 'available from the corresponding author upon reasonable request,' which does not constitute release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, Docker setup, or dependency versions are provided. The paper does not specify which BERT variant was used, what libraries were employed, or any technical setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The paper describes the system architecture conceptually but provides no steps for reproducing the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Only point estimates are reported (e.g., accuracy of 0.99, 0.98, 0.91). No confidence intervals or error bars are provided for any metric."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Performance differences across use cases (e.g., 0.99 vs 0.91) are presented as raw numbers without any testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. Results are presented as absolute accuracy, precision, recall, and F1 values without any baseline context or effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why ~4 million training data points or ~250k validation records were chosen. The validation set sizes per use case (221k, 4.1k, 1.1k, 13.4k) are stated without justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be from single runs with no indication of result stability across multiple experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baseline comparisons are included. The BERT model is evaluated in isolation without comparison to any other prompt injection detection method, rule-based system, or alternative ML model."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included at all, so they cannot be assessed for contemporariness. The paper does not compare against any existing prompt injection detection approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has three components (banned terms, vector embeddings, BERT model) but no ablation study is performed. Only the BERT component is evaluated; the contribution of each component to the overall system is not measured."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports accuracy, F1 score, precision, and recall (Figure 4), as well as the full confusion matrix (TN, FP, FN, TP) in Table 5."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the detection system's outputs is performed. All evaluation is automated using labeled datasets."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 states the model was 'benchmarked against an unseen validation dataset comprising around 250k records,' indicating a held-out evaluation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across four use cases (General, Product Query, AI Assistant, Question Answering) in Table 4 and Figure 4, showing per-category performance."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes the AI Assistant use case achieved only 0.91 accuracy 'due to lack of diverse data' but provides no specific failure examples, error analysis, or discussion of what types of inputs the system misclassifies."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The AI Assistant use case is reported at 0.91 accuracy (vs 0.98-0.99 for others), and Section 4.1 acknowledges that 'a high false positive rate could detrimentally affect usability.' The Product Query use case shows precision of only ~0.41."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the system 'identify and mitigate prompt injections effectively' and aims to 'neutralize prompt injections in real-time.' However, only the BERT component is evaluated. The banned terms and vector embedding components are described conceptually but never tested, so the claim about the combined system is unsupported."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not make explicit causal claims. It reports classification performance metrics without claiming that specific components cause specific improvements."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Prompt Injection Detection in LLM Integrated Applications' broadly, but evaluation is limited to text classification with a single BERT model on specific datasets. No bounds are stated regarding which LLMs, application types, or attack categories the approach generalizes to."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. For example, the high accuracy on the General dataset could be due to class imbalance (Table 5 shows TN=0, FP=0, meaning no negative examples), but this is not acknowledged."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures binary classification accuracy on labeled datasets but frames this as 'neutralizing prompt injections in real-time' and 'safeguarding integrity and security.' The gap between detecting known patterns in labeled data and preventing real-world prompt injections in production systems is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'BERT model' and 'pretrained BERT model' throughout without specifying which variant (e.g., bert-base-uncased, bert-large, DistilBERT), version, or checkpoint was used."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The core approach is BERT fine-tuning for classification, not prompt-based. The system does not use prompting as its detection mechanism."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for the BERT fine-tuning process. Learning rate, batch size, number of epochs, optimizer, and other training settings are entirely absent."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a classification pipeline, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists data sources [44-56] and total sizes (~4M training, ~250k validation) but does not document how data from 12+ sources was combined, deduplicated, cleaned, or split into training and validation sets."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "Section 4 ('Discussions') covers challenges and future work but does not serve as a limitations section for the current study. The discussion is about general challenges (balancing security/usability, evolving threats) rather than limitations of the presented evaluation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. Section 4 describes generic challenges in the domain but does not address specific threats to this study's validity, such as dataset bias, class imbalance, or limited evaluation scope."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what the results do NOT show or what settings are excluded from their claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data is not available. The open-source datasets are individually accessible, but the combined training/validation data and internal eBay product data are not released. The Data Availability Statement offers data 'upon reasonable request.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Data sources are listed by reference ([44-56]) and total size (~4M) is stated, but the actual collection procedure — how data was gathered from each source, what criteria were used for inclusion, and how sources were combined — is not described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public datasets and internal product data."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw data sources to final evaluation is undocumented. How ~4M records were processed, labeled, and split into training and validation sets is not described. Table 5 totals do not match Table 4 dataset sizes (e.g., Question Answering: 13.4k in Table 4 but 11,446 in Table 5)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'This research received no external funding.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are identified as being from 'Global Information Security, eBay Inc. San Jose, CA, USA.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "While no external funding is disclosed, all authors work at eBay's security team, and the system is designed for eBay's products. eBay has a direct commercial interest in the detection system working well."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes 'Conflicts of Interest: The authors declare no conflict of interest.' A competing interests statement is present, though it does not acknowledge the inherent interest eBay has in the system's effectiveness."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests a defense system (prompt injection detection via fine-tuned BERT), not a pre-trained model's capability on a benchmark. Contamination in the pre-training sense is not applicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper evaluates a fine-tuned detection model, not a pre-trained LLM's benchmark performance."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — contamination of pre-training data is not applicable to this fine-tuned classification task."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'latency is within milliseconds' but provides no specific measurements of their own system's inference time. Table 2 shows Pinecone vector database latency benchmarks from a third party, not their own system's performance."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget is stated. Training time, GPU hours, hardware used, or any computational resource information for fine-tuning the BERT model is absent."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Results appear to be from a single training run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The paper does not mention how the BERT model's hyperparameters were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the final model configuration was selected. Only final results are shown without explaining the selection process."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system exclusively without acknowledging the bias inherent in self-evaluation. No independent evaluation or baseline comparison is provided."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "Only one model configuration is tested, so compute-performance tradeoff analysis is not applicable."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the datasets used actually measure real-world prompt injection detection ability. The General validation set has TN=0 and FP=0 (Table 5), meaning it contains no negative examples, yet 0.99 accuracy is claimed without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The system is a classification pipeline."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and validation data overlap temporally. Data sources span 2022-2024 with no temporal split analysis."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether input features could leak label information. Given 12+ data sources combined, potential feature leakage between overlapping datasets is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of train/test independence. Multiple data sources from HuggingFace may contain overlapping or near-duplicate examples, and this is not verified."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is described. No deduplication, overlap analysis, or temporal splits are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "The fine-tuned BERT model achieves accuracy exceeding 0.98 on prompt injection detection across use cases.",
       "evidence": "Table 4 shows accuracy of 0.99 (General, 221k), 0.99 (Product Query, 4.1k), 0.91 (AI Assistant, 1.1k), and 0.98 (Question Answering, 13.4k). Section 3.4.",
       "supported": "weak"
     },
     {
       "claim": "The multi-faceted detection system combining banned terms, vector embeddings, and BERT can neutralize prompt injections in real-time.",
       "evidence": "Only the BERT component is empirically evaluated (Section 3.4). Banned terms and vector embeddings are described conceptually (Sections 3.1, 3.2) but not tested. Latency claim is vague ('within milliseconds') with no system-level measurement.",
       "supported": "unsupported"
     },
     {
       "claim": "The BERT model shows high recall and identifies most prompt injections across all use cases.",
       "evidence": "Table 5 and Figure 4 show recall rates. General: 0.999, Product Query: 1.0, AI Assistant: 0.984, Question Answering: 0.997. However, the General dataset has TN=0, meaning no negative examples are present.",
       "supported": "weak"
     },
     {
       "claim": "The detection system is suitable for production deployment in data-intensive applications.",
       "evidence": "Section 4.1 discusses balancing security and usability conceptually. No production deployment metrics, A/B tests, or real-world performance data are provided.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Confusion matrix anomalies and class imbalance",
       "detail": "Table 5 shows the General validation set has TN=0 and FP=0, meaning it contains zero negative (non-injection) examples. The 0.99 accuracy is therefore just recall on an all-positive dataset. This severely undermines the accuracy claims."
     },
     {
       "flag": "Dataset size discrepancies between tables",
       "detail": "Table 4 lists Question Answering at 13.4k records but Table 5 totals (5555+168+17+5706) sum to 11,446, a discrepancy of ~2,000 records with no explanation."
     },
     {
       "flag": "No baselines or comparisons",
       "detail": "The BERT model is evaluated in isolation without comparison to any baseline method (rule-based, other ML models, existing prompt injection detectors). This makes it impossible to assess whether the approach offers value over simpler alternatives."
     },
     {
       "flag": "Two-thirds of the system is unevaluated",
       "detail": "The paper proposes a three-component system (banned terms, vector embeddings, BERT) but only evaluates the BERT component. The overall system effectiveness claimed in the abstract is therefore unsupported."
     },
     {
       "flag": "Company evaluating its own product",
       "detail": "All authors are from eBay's security team, building and evaluating a system designed for eBay's internal use. No independent evaluation is provided, and the conflicts of interest statement does not acknowledge this inherent bias."
     },
     {
       "flag": "Missing training details",
       "detail": "The BERT variant, hyperparameters, training procedure, data splitting methodology, and preprocessing steps are entirely absent. The paper is effectively unreproducible."
     }
   ],
   "cited_papers": [
     {
       "title": "Prompt injection attack against LLM-integrated applications",
       "authors": ["Yi Liu", "Gelei Deng", "Yuekang Li"],
       "year": 2023,
       "arxiv_id": "2306.05499",
       "relevance": "Foundational work on prompt injection attacks against LLM-integrated applications, directly relevant to the threat model this paper addresses."
     },
     {
       "title": "From prompt injections to SQL injection attacks: How protected is your LLM-integrated web application?",
       "authors": ["Rodrigo Pedro", "Daniel Castro", "Paulo Carreira"],
       "year": 2023,
       "arxiv_id": "2308.01990",
       "relevance": "Explores the connection between prompt injection and traditional injection attacks in LLM-integrated web applications."
     },
     {
       "title": "Assessing prompt injection risks in 200+ custom GPTs",
       "authors": ["Jiahao Yu", "Yuhang Wu", "Dong Shu"],
       "year": 2023,
       "arxiv_id": "2311.11538",
       "relevance": "Large-scale assessment of prompt injection vulnerabilities across custom GPT applications."
     },
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra"],
       "year": 2023,
       "doi": "10.1145/3605764.3623985",
       "relevance": "Demonstrates indirect prompt injection attacks against real-world LLM-integrated applications."
     },
     {
       "title": "More than you've asked for: A comprehensive analysis of novel prompt injection threats to application-integrated large language models",
       "authors": ["Kai Greshake", "Sahar Abdelnabi", "Shailesh Mishra"],
       "year": 2023,
       "arxiv_id": "2302.12173",
       "relevance": "Comprehensive taxonomy of prompt injection threats to LLM-integrated applications."
     },
     {
       "title": "Automatic and universal prompt injection attacks against large language models",
       "authors": ["Xiaogeng Liu", "Zhiyuan Yu", "Yizhe Zhang"],
       "year": 2024,
       "arxiv_id": "2403.04957",
       "relevance": "Proposes automated methods for generating universal prompt injection attacks against LLMs."
     },
     {
       "title": "A new era in LLM security: Exploring security concerns in real-world LLM-based systems",
       "authors": ["Fangzhou Wu", "Ning Zhang", "Somesh Jha"],
       "year": 2024,
       "arxiv_id": "2402.18649",
       "relevance": "Surveys security concerns in real-world LLM-based systems including data extraction and prompt injection."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["Andy Zou", "Zifan Wang", "Nicholas Carlini"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Demonstrates universal adversarial attacks on aligned LLMs, relevant to understanding attack vectors."
     },
     {
       "title": "Jailbroken: How does LLM safety training fail?",
       "authors": ["Alexander Wei", "Nika Haghtalab", "Jacob Steinhardt"],
       "year": 2023,
       "relevance": "Analyzes failure modes of LLM safety training, including techniques that bypass alignment."
     },
     {
       "title": "Demystifying RCE vulnerabilities in LLM-integrated apps",
       "authors": ["Tong Liu", "Zizhuang Deng", "Guozhu Meng"],
       "year": 2023,
       "arxiv_id": "2309.02926",
       "relevance": "Examines remote code execution vulnerabilities in LLM-integrated applications."
     },
     {
       "title": "Large language models for code: Security hardening and adversarial testing",
       "authors": ["Jingxuan He", "Martin Vechev"],
       "year": 2023,
       "doi": "10.1145/3576915.3623175",
       "relevance": "Addresses security hardening of LLMs for code generation, relevant to LLM security and adversarial robustness."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Presents a production-oriented architecture for prompt injection detection with specific deployment patterns, but no released tool or code to use directly."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Uses standard approaches (banned terms + embeddings + BERT) with no surprising findings or challenges to conventional wisdom."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Addresses prompt injection as a security concern but focuses on defense rather than demonstrating novel attacks or raising new risks."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward presentation of a detection system."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No public code, demo, or tool released that someone could try."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "eBay is a recognizable company but not a leading AI research lab."
     }
   }
 }