ai-research-survey

scan.json (28469B)

---

 {
   "paper": {
     "title": "Design and Implementation of a Secure RAG-Enhanced AI Chatbot for Smart Tourism Customer Service: Defending Against Prompt Injection Attacks – A Case Study of Hsinchu, Taiwan",
     "authors": [
       "Yu-Kai Shih",
       "You-Kai Kang"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2509.21367",
     "doi": "10.48550/arXiv.2509.21367"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "case-study",
     "benchmark-eval"
   ],
   "key_findings": "A RAG-enhanced tourism chatbot with layered defenses (system norms, gatekeeper, reverse RAG) blocks 301 of 674 adversarial injection prompts (45% recall) while maintaining 95% benign accuracy. GPT-5 without guardrails blocks only 249/674 attacks (37% recall), demonstrating that base model improvements alone are insufficient. The Gatekeeper (V2) and Reverse RAG (V3) versions achieve identical block counts across all attack categories, suggesting the reverse RAG layer adds no measurable incremental defense value. Response latency increases from 2.1s (baseline) to 3.2s (secure RAG) with the added defense layers.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or download link is provided anywhere in the paper. The system was developed by a Taiwan-based tourism technology firm but no source code is released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The adversarial datasets reference public sources (Deepset, Rubend18) but the 223 benign test queries and 49 partner-provided adversarial samples are not released. No download link for any combined dataset is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.4 lists 'Linux workstation with a single modern NVIDIA GPU, 64 GB RAM' and 'Python 3.11; Flowise; Qdrant; LangSmith' but provides no requirements.txt, Dockerfile, or detailed library versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The paper describes the architecture conceptually but does not provide enough detail to replicate the system."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3-5 are reported as point estimates only. No confidence intervals, error bars, or uncertainty measures appear anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims differences between defense versions (e.g., V0 vs V3) and between GPT-4o and GPT-5 based solely on comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides baseline context for all metrics (e.g., benign accuracy 78%→95%, hallucination rate 15%→2%, injection block rate 0%→100%), allowing the reader to assess the magnitude of improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the 223 benign queries or 674 adversarial prompts. No power analysis or sample size rationale is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across runs, or any spread measures are reported. Results appear to be from single runs with no indication of result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "V0 (zero defense) serves as a naive baseline, with V1 (system norms), V2 (gatekeeper), V3 (reverse RAG), and V4 (GPT-5 direct) compared incrementally in Tables 3-5."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "All baselines are internal variants of the same system. No comparison is made against any published prompt injection defense system, external guardrail framework, or competing approach from the literature."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The V0→V1→V2→V3 progression effectively ablates defense layers, and V4 (GPT-5 direct) tests model capability with minimal scaffolding. Tables 3-4 show the incremental contribution of each layer."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports precision, recall, accuracy, and F1 for attack detection. Table 5 reports benign accuracy, hallucination rate, injection block rate, response time, and user satisfaction."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 reports 'User Satisfaction (1-5)' scores (3.4 to 4.8) but provides zero description of the evaluation methodology — no number of evaluators, no evaluation protocol, no questionnaire, no recruitment process."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit separation between development/tuning data and test data is described. It is unclear whether any of the 223 benign queries or 674 adversarial prompts influenced the defense design during development."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 breaks down blocked attacks by attack type (double character, virtualization, obfuscation, payload splitting, adversarial suffix, instruction manipulation) across all defense versions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The 'Failure Case Analysis' section discusses three representative failure modes: indirect obfuscation with benign wrappers, multi-turn anchoring, and ambiguous safety scopes, with proposed mitigations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper honestly reports that overall recall is only 45% (373/674 attacks missed), that GPT-5 blocks only 37% on the full corpus, and that V2 and V3 achieve identical block counts (suggesting V3's reverse RAG adds no measurable value)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the secure version achieves 'substantial detection and mitigation rates across attack categories,' but actual recall on the full 674-attack corpus is only 45% (Table 4). The abstract's framing significantly overstates the system's defensive capability."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The V0→V1→V2→V3 incremental ablation design is a controlled single-variable manipulation — each version adds one defense layer while keeping others constant. This is adequate for the causal claims about each layer's contribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title bounds results to Hsinchu, but the conclusion claims the system 'serves as a practical blueprint for deploying secure AI in visitor services' and aims to 'foster innovation in secure smart tourism globally.' A system with 45% attack detection rate does not support 'blueprint' claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for results. For example, no consideration that the 301 blocked attacks may simply be the easiest/most formulaic attacks, or that the defense pattern-matches known attack templates rather than genuinely understanding intent."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames 45% recall as 'substantial' defense and presents 'User Satisfaction' scores without explaining what they measure or how they were collected. The gap between the proxy (block rate on curated attack sets) and the claimed outcome (system security) is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o' and 'GPT-5 (released 2025-08-07)' but provides no snapshot dates or API version identifiers. Marketing names without snapshot dates do not constitute specified versions per the schema."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Sections 3.2.2 and 3.2.4 provide prompt 'excerpts' and 'skeletons' but not complete prompts. The system norms prompt is labeled 'Prompt Skeleton (excerpt)' and the summary directive is a fragment. The full prompt text used in experiments is not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 reports: 'Retrieval top-k = 5; similarity threshold τ = 0.70; temperature = 0.2; max tokens = 1024; cosine distance for relevance.'"
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The Agentflow variants (V0-V4) are described in detail in Sections 3.2.1-3.2.5 with workflow diagrams (Figures 1-7), component descriptions (intent router, relevance check, leak scan), and decision tables (Table 1)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how the adversarial datasets were combined, filtered, or preprocessed. The 373 attacks that don't fall into any of the 6 taxonomy categories (674 total minus 301 categorized) are never explained or characterized."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations' provides substantive discussion across 9 specific limitation areas including dataset diversity, API vulnerabilities, RAG scope, multilingual coverage, and adversarial threat evolution."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 lists specific threats: 'Internal queries may not capture full diversity; field trials are essential,' 'Tested known attacks; emerging threats (e.g., indirect injections) may evade,' and 'GPT-5: Early access limited testing depth.' These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states specific exclusions: multilingual support is 'limited to major languages; slang/dialects unhandled,' 'Flowise limits flexibility,' and adversarial testing covered only 'known attacks' with emerging threats unexplored."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is released. The 223 benign queries, 49 partner-provided adversarial samples, system responses, and CPE logs are not available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.3 names the adversarial sources ('Deepset (546), Rubend18 (79), partner (49, adapted)') and states '223 benign queries (informational/transactional/exploratory)' but does not describe how the benign queries were created, how partner samples were adapted, or what inclusion criteria were used."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Data sources are adversarial prompt datasets and system-generated test queries, not human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of the data pipeline from raw datasets to final evaluation. The 674 total adversarial prompts include 373 that are not categorized into any attack type in Table 2, and this gap is never explained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section or acknowledgments mentioning financial support. The paper references development by 'a Taiwan-based tourism technology firm' but does not disclose the financial arrangement or name the firm."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "Author affiliations list 'National Dong Hwa University' and 'BTS Experimental Education Program' but the relationship between the authors and the unnamed tourism tech firm that developed the system is not disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The unnamed tourism technology firm that developed the chatbot has a clear commercial interest in the system being shown as effective. This conflict is not acknowledged or discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement appears anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests defense mechanisms against prompt injection attacks rather than evaluating a pre-trained model's knowledge on a benchmark. The adversarial prompts test security behavior, not learned knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a defense-testing study (red-teaming) that evaluates guardrails against adversarial inputs, not a benchmark evaluation of model knowledge."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation tests whether defense layers block attacks, not whether the model has memorized benchmark answers. Contamination in the traditional sense does not apply."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Evaluation is entirely automated using adversarial and benign prompt datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study tests an AI chatbot system against automated datasets."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All evaluation is automated."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or evaluators requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports response time in seconds for each version: Baseline 2.1s, RAG 2.8s, Secure RAG 3.2s, GPT-5 Direct 2.5s. This provides wall-clock latency per query."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.4 mentions hardware ('Linux workstation with a single modern NVIDIA GPU, 64 GB RAM') but does not quantify total API spend, GPU hours, or computational budget for the evaluation."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple runs, random seeds, or seed sensitivity analysis. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search described. The similarity threshold τ=0.70 and temperature=0.2 appear chosen without justification or search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the hyperparameter values (τ=0.70, top-k=5, temperature=0.2) were selected. No validation set or selection methodology described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparisons across 5 system versions and 6+ attack categories without any statistical tests, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All baselines are the authors' own system variants. No acknowledgment that evaluating their own system against their own baselines may introduce bias, and no independent evaluation is conducted."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Response times are reported per version (2.1-3.2s in Table 5) but there is no systematic analysis of compute-performance tradeoffs or discussion of whether added latency is justified by defense gains."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the Deepset, Rubend18, and partner datasets are valid proxies for real-world prompt injection threats. The Deepset dataset includes 'politics/role-play' samples whose relevance to prompt injection is unclear."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The GPT-5 ablation (V4) uses GPT-5 with minimal scaffolding (no V2 relevance gating) while V3 uses GPT-4o with full guardrails. This conflates model and scaffolding effects, but the confound is not discussed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Deepset and Rubend18 adversarial datasets are publicly available and may have been included in GPT-4o/GPT-5 training data. This potential temporal leakage is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides any hints or features that would not be available in real-world deployment scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether attacks within or across datasets share structural similarities that could inflate apparent defense performance."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. The paper does not check whether the adversarial prompts appear in model training data."
       }
     }
   },
   "claims": [
     {
       "claim": "RAG integration improves benign query accuracy from 78% to 95%.",
       "evidence": "Table 5 shows baseline benign accuracy of 78% vs RAG and Secure RAG at 95%. GPT-5 Direct achieves 96%.",
       "supported": "moderate"
     },
     {
       "claim": "The Secure RAG version achieves 100% injection block rate.",
       "evidence": "Table 5 claims 100% injection block rate, but the footnote clarifies this is on 'a focused subset of 301 high-confidence injection samples.' Table 4 shows actual recall of 45% on the full 674-attack corpus.",
       "supported": "weak"
     },
     {
       "claim": "GPT-5 blocks approximately 85% of attacks without additional guardrails.",
       "evidence": "Table 5 reports 85% injection block rate for GPT-5 Direct. However, Table 4 shows GPT-5 blocked only 249/674 attacks (37% recall). The 85% figure appears inconsistent with the full-corpus results.",
       "supported": "weak"
     },
     {
       "claim": "Layered defenses progressively improve attack detection from F1 0.01 (V0) to F1 0.62 (V3).",
       "evidence": "Table 4 shows F1 progression: V0=0.01, V1=0.48, V2=0.62, V3=0.62. However, V2 and V3 achieve identical scores, indicating the Reverse RAG layer adds no measurable defense value.",
       "supported": "moderate"
     },
     {
       "claim": "RAG reduces hallucination rate from 15% to 2%.",
       "evidence": "Table 5 reports hallucination rates: Baseline 15%, RAG 2%, Secure RAG 2%, GPT-5 Direct 1%. No methodology for measuring hallucinations is described.",
       "supported": "weak"
     },
     {
       "claim": "Multi-layered linguistic analysis across lexical, semantic, intentional, contextual, and pragmatic levels enables accurate intent detection.",
       "evidence": "Section 3.1 describes five parsing levels conceptually. No empirical evaluation isolates the contribution of linguistic analysis to intent detection accuracy.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Misleading 100% block rate claim",
       "detail": "Table 5 claims 100% injection block rate for Secure RAG, but the footnote reveals this is only on 301 of 674 attacks (45% of the corpus). The abstract similarly describes 'substantial detection and mitigation rates' for 45% recall. This framing materially misrepresents the system's actual defensive capability."
     },
     {
       "flag": "373 unaccounted adversarial samples",
       "detail": "The paper claims 674 total adversarial prompts but the attack taxonomy in Table 2 only accounts for 301 (23+184+71+0+4+19). The remaining 373 attacks — which the system fails to block — are never categorized, explained, or analyzed."
     },
     {
       "flag": "Inconsistent GPT-5 metrics across tables",
       "detail": "Table 5 reports 85% injection block rate for GPT-5 Direct, but Table 4 shows only 249/674 blocked (37% recall). The discrepancy is not explained, and the footnote only addresses the Secure RAG 100% figure."
     },
     {
       "flag": "User satisfaction scores without methodology",
       "detail": "Table 5 reports user satisfaction scores (3.4 to 4.8 on a 1-5 scale) with zero description of evaluation methodology — no evaluator count, no evaluation protocol, no questionnaire design, no recruitment process."
     },
     {
       "flag": "V2 and V3 identical results unexplained",
       "detail": "The Gatekeeper (V2) and Reverse RAG (V3) versions block identical numbers of attacks across every category in Table 3 (both total 301). The Reverse RAG layer — presented as a key contribution — adds zero measurable defensive value, but this is not discussed."
     },
     {
       "flag": "No external defense baselines",
       "detail": "All comparisons are between internal system variants. No published prompt injection defense system, external guardrail framework, or competing approach from the literature is compared against."
     },
     {
       "flag": "Undisclosed commercial conflict of interest",
       "detail": "The paper describes development by 'a Taiwan-based tourism technology firm' that has a commercial interest in the system being shown as effective. The firm is not named, the financial arrangement is not disclosed, and no competing interests statement appears."
     }
   ],
   "cited_papers": [
     {
       "title": "Guardrails for large language models: A review of techniques and challenges",
       "authors": ["Akheel"],
       "year": 2025,
       "relevance": "Directly surveys LLM guardrail techniques and challenges, relevant to the survey's coverage of prompt injection defenses."
     },
     {
       "title": "An early categorization of prompt injection attacks on large language models",
       "authors": ["Sippo Rossi", "Alisia Marianne Michel", "Raghava Rao Mukkamala", "Jason Bennett Thatcher"],
       "year": 2024,
       "arxiv_id": "2402.00898",
       "relevance": "Provides a foundational taxonomy of prompt injection attacks on LLMs, directly relevant to the survey's prompt injection coverage."
     },
     {
       "title": "LLM01:2025 Prompt Injection",
       "authors": ["OWASP"],
       "year": 2025,
       "relevance": "OWASP's official prompt injection risk classification for LLMs, a key reference standard for AI security research."
     },
     {
       "title": "Prompt Injection: An Analysis of Recent LLM Security Incidents",
       "authors": ["NSFOCUS"],
       "year": 2025,
       "relevance": "Analyzes real-world LLM security incidents involving prompt injection, relevant to understanding practical attack vectors."
     },
     {
       "title": "zIA: a GenAI-powered personalized local assistant assists tourists in Italy",
       "authors": ["A. Cassani", "M. Ruberl", "A. Salis", "G. Boanelli", "G. Giannese"],
       "year": 2025,
       "relevance": "Closely related GenAI tourism assistant system, useful for comparative analysis of AI chatbot architectures."
     },
     {
       "title": "Enhancing tourism recommender systems for sustainable city trips using retrieval-augmented generation",
       "authors": ["A. Banerjee", "A. Satish", "W. Wörndl"],
       "year": 2025,
       "doi": "10.1007/978-3-031-87654-7_3",
       "relevance": "RAG-based tourism recommendation system, relevant to understanding RAG applications in domain-specific AI deployments."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Describes a deployable chatbot architecture with defense layers that practitioners could adapt, though no code or tools are released."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected finding that layered defenses help and that base models alone are insufficient against prompt injection."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Shows that even with multi-layered defenses, 55% of adversarial attacks succeed, and GPT-5 alone blocks only 37% of the full attack corpus."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenges to established players or widely-held beliefs."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code repository, no demo, no downloadable tool or dataset released."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "References GPT-5 and GPT-4o (recognizable OpenAI products) but the paper is from an unknown university program, not a major lab."
     }
   }
 }