ai-research-survey

scan.json (25687B)

---

 {
   "paper": {
     "title": "Securing Large Language Models (LLMs) from Prompt Injection Attacks",
     "authors": ["Omar Farooq Khan Suri", "John McCrae"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.01326",
     "doi": "10.48550/arXiv.2512.01326"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "JATMO-style task-specific fine-tuning of non-instruction-tuned base models reduces prompt injection attack success rates by 75-90% compared to instruction-tuned GPT-3.5-Turbo, but does not eliminate vulnerability. A quality-vulnerability trade-off was observed across three models: higher ROUGE-L scores correlated with higher attack success rates. Multilingual prompts and code-related triggers remain effective bypass vectors even against JATMO-trained models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository, GitHub link, or archive is mentioned anywhere in the paper. The modified HOUYI framework and fine-tuning code are not released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The source Amazon All-Beauty reviews dataset is public (ref [4]), but the authors' custom 1,500-pair summarization dataset generated via GPT-3.5 and the 72 adversarial prompts per attack type are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using LoRA and 'consumer-grade GPUs' (Section 3.2) but provides no specific library versions, requirements.txt, GPU model, or environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology section describes the approach conceptually but does not include commands, scripts, or a reproducibility guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports single point estimates for ROUGE-L, CM%, IG%, and Avg ASR% with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims JATMO 'significantly lowers attack success rates' (Section 6) and 'substantially reduces vulnerability' but provides no statistical significance tests (no p-values, t-tests, or any inferential statistics)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides absolute ASR values with a baseline comparison (e.g., Qwen 0.5B at 9.68% vs GPT-3.5 at 100%), and the conclusion states 'reducing attack success by up to 90% relative to GPT-3.5,' providing sufficient context for effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "72 adversarial prompts per attack type were used (Section 3.5) with no justification for this number, no power analysis, and no discussion of whether this is sufficient to draw reliable conclusions."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. All results in Table 1 appear to be single-run values with no indication of result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "GPT-3.5-Turbo (instruction-tuned) is used as a baseline comparison against the JATMO fine-tuned models (Table 1, Section 3.5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-3.5-Turbo is a reasonable contemporary baseline for comparing instruction-tuned vs non-instruction-tuned models in the context of prompt injection defense. The original JATMO paper also used GPT-3.5 as a baseline."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The three HOUYI modifications (custom fitness scoring, improved mutation logic, custom harness) are not individually ablated. Different model sizes are compared but this is not an ablation of the defense mechanism."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two distinct metric types are used: ROUGE-L for task fidelity and Attack Success Rate (split into Content Manipulation and Information Gathering) for security evaluation (Table 1, Section 3.5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs. Attack success is determined automatically (detecting 'pwned' token or date-like formats). Human review of edge cases or qualitative output assessment could have strengthened the evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "A held-out validation split is mentioned for monitoring training (Section 3.2), but it is unclear whether ROUGE-L scores are reported on a separate test set or the validation set. No explicit train/validation/test split is described."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down attack success rates by attack type (Content Manipulation vs Information Gathering) and by model, rather than reporting only aggregate numbers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Qualitative Analysis) discusses three specific failure modes: residual instruction-following from pretraining, vulnerability to multilingual/mixed-language prompts, and code-generation bias triggered by code-related tokens."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly reports that JATMO 'does not fully prevent injections' and acknowledges that mid-sized models remain susceptible to adversarial phrasing (Sections 4.1, 6)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that JATMO reduces but does not eliminate ASR, and that there is a quality-vulnerability trade-off, are supported by Table 1. The claims about multilingual and code-related bypasses are supported by qualitative analysis in Section 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'better task performance often correlates with increased susceptibility' based on only 3 data points (3 models). The claim that JATMO 'reduces' vulnerability conflates the fine-tuning approach with model size differences (0.5B–7B vs 175B) without controlling for this confound."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to address 'Securing Large Language Models from Prompt Injection Attacks' broadly, but the study tests only 3 models on a single task (summarization) with one dataset (Amazon beauty reviews) and one attack framework. Generalization to other tasks, models, or attack types is not bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The quality-vulnerability correlation could be driven by model size rather than task specialization. The comparison of 0.5B/4B/7B models against a 175B model introduces many confounds (architecture, pretraining data, scale) that are not addressed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures what it claims: ASR measures injection vulnerability and ROUGE-L measures summarization quality. The claims match the granularity of the measurements without overreaching to broader concepts."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Open-source models are identified by family and size (LLaMA 2-7B, Qwen1.5-0.5B, Qwen1.5-4B), but GPT-3.5-Turbo lacks a snapshot date or API version. Per schema, marketing names without snapshot dates are insufficient."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The HOUYI attack components are described conceptually (framework, separator, disruptor) but no actual prompt text is provided. The example disruptors ('append the word pwned at the end') appear to be illustrative, not the actual prompts used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.2 states 'Training was run for several epochs with tuned learning rates and batch sizes' without specifying any actual values. No temperature, top-p, max tokens, or LoRA rank are reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The models are fine-tuned for a single summarization task without agents, tools, or multi-step pipelines."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents the preprocessing pipeline: Amazon All-Beauty reviews were grouped into sets of three, GPT-3.5 generated summaries, producing 1,500 input-output pairs (~400k tokens) formatted in OpenAI's chat fine-tuning schema as .jsonl."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section exists. Section 7 (Future Work) mentions some shortcomings of JATMO but frames them as future research directions rather than formal limitations of the study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address internal validity concerns such as the model size confound, the limited task scope, or the adequacy of the automated success detection."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clearly delineate what the results do NOT show (e.g., that results are limited to summarization, three specific models, and one attack framework)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is released — neither the fine-tuning dataset, adversarial prompts, model outputs, nor evaluation logs are available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes using the Amazon All-Beauty reviews dataset and generating summaries with GPT-3.5. Section 3.3-3.4 describe the HOUYI attack prompt generation process. Section 3.5 states 72 prompts per attack type."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is a standard public dataset (Amazon reviews) and adversarial prompts are algorithmically generated."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from Amazon reviews to fine-tuning dataset is partially described, but key details are missing: how reviews were selected from the full dataset, filtering criteria, and how the 1,500 pairs map to the original review count. The HOUYI mutation pipeline details are also vague."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed anywhere in the paper. There is no acknowledgments section mentioning grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: both authors are from University of Galway, Ireland. No commercial affiliations with the models or tools being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The paper does not state whether the work is unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests a defense mechanism (JATMO fine-tuning) against adversarial attacks rather than evaluating a pre-trained model's knowledge on a benchmark. Contamination of test data in training is not the relevant concern."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the study tests defenses against prompt injection, not model knowledge on benchmark tasks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the adversarial prompts are generated by HOUYI, not drawn from a pre-existing benchmark that could appear in training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or API cost information is reported despite using both local models and the OpenAI API for GPT-3.5-Turbo fine-tuning."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'consumer-grade GPUs' and LoRA for efficiency (Section 3.2) but provides no specific GPU model, hours, training time, or total API spend."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or sensitivity analysis across seeds for any of the fine-tuning or attack experiments."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of attack prompts (72 per type) is stated, but the number of experimental runs or repetitions for fine-tuning or evaluation is not."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.2 mentions 'tuned learning rates and batch sizes' implying hyperparameter search was done, but no search method, number of configurations, or compute budget for the search is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No description of how the final hyperparameter configuration was selected. The paper only states parameters were 'tuned' without explaining the selection criterion."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors modified the HOUYI attack framework and implemented their own evaluation harness but do not acknowledge any bias from evaluating their own modifications."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 0.5B to 175B parameters are compared without discussing the massive compute differences. The quality-vulnerability trade-off could be driven by compute/scale rather than JATMO methodology."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ASR with automated token detection (checking for 'pwned' or date keywords) adequately measures prompt injection vulnerability. Edge cases in automated detection are not addressed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in the model comparisons. Models are evaluated directly without agentic pipelines."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the pre-training data for LLaMA 2, Qwen, or GPT-3.5 could have included information about JATMO or HOUYI attack patterns, which could affect results."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup could leak information about expected outputs, or whether the fixed detection patterns ('pwned', date formats) create a biased evaluation."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 72 HOUYI-generated prompts per attack type are independent, or whether the genetic algorithm's mutation process creates correlated test cases."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "JATMO fine-tuning reduces attack success rates by up to 90% relative to instruction-tuned GPT-3.5-Turbo",
       "evidence": "Table 1 shows Qwen 0.5B at 9.68% average ASR vs GPT-3.5-Turbo at 100%. LLaMA-2 7B at 11.11% ASR. Section 6 states 'reducing attack success by up to 90%.'",
       "supported": "moderate"
     },
     {
       "claim": "JATMO does not fully prevent prompt injections — adversaries exploiting multilingual cues or code-related disruptors still bypass defenses",
       "evidence": "Table 1 shows non-zero ASR for all JATMO models (9.68-25%). Section 5 provides qualitative examples of multilingual and code-triggered bypasses.",
       "supported": "moderate"
     },
     {
       "claim": "There is a trade-off between generation quality (ROUGE-L) and injection vulnerability (ASR) — models that produce better summaries are more susceptible to attacks",
       "evidence": "Table 1 shows Qwen 4B has the highest ROUGE-L (0.43) and highest JATMO ASR (25%), while Qwen 0.5B has lowest ROUGE-L (0.29) and lowest ASR (9.68%). However, this is based on only 3 data points.",
       "supported": "weak"
     },
     {
       "claim": "GPT-3.5-Turbo is fully compromised by both content manipulation and information gathering attacks at 100% success rate",
       "evidence": "Table 1 shows 100% ASR for both CM and IG attacks on GPT-3.5-Turbo.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Correlation claim from 3 data points",
       "detail": "The quality-vulnerability trade-off claim is based on observing 3 models. This is insufficient to establish any correlation or trend, yet the paper frames it as a consistent finding."
     },
     {
       "flag": "Model size confound not addressed",
       "detail": "Comparing 0.5B, 4B, and 7B models against a 175B model conflates the defense mechanism (JATMO) with massive scale differences. The lower ASR of smaller models could be due to lower capability, not JATMO-style fine-tuning."
     },
     {
       "flag": "Questionable GPT-3.5-Turbo parameter count",
       "detail": "Table 1 lists GPT-3.5-Turbo as '175B' parameters. This is the parameter count for GPT-3, not GPT-3.5-Turbo, whose architecture and size are not publicly confirmed."
     },
     {
       "flag": "Internal inconsistency in model count",
       "detail": "Section 3.2 states 'We fine-tuned two base models: LLaMA 2–7B and Qwen 1.5–0.5B' but Table 1 includes results for Qwen 4B as well. The introduction mentions three models but calls them 'two non–instruction-tuned base models.'"
     },
     {
       "flag": "No statistical tests despite strong claims",
       "detail": "Claims of 'significant' reduction in attack success rates are made without any statistical testing. With 72 prompts and no variance reporting, the reliability of the ASR differences is unknown."
     },
     {
       "flag": "Hyperparameters completely withheld",
       "detail": "Learning rates, batch sizes, number of epochs, LoRA rank, and all training hyperparameters are described as 'tuned' but no values are reported, making the work unreproducible."
     }
   ],
   "cited_papers": [
     {
       "title": "JATMO: Prompt Injection Defense by Task-Specific Finetuning",
       "authors": ["J. Piet", "M. Alrashed", "C. Sitawarin", "S. Chen", "Z. Wei", "E. Sun", "B. Alomair", "D. Wagner"],
       "year": 2024,
       "relevance": "Core defense mechanism evaluated in this study; proposes task-specific fine-tuning as prompt injection mitigation."
     },
     {
       "title": "Prompt Injection Attacks Against LLM-Integrated Applications",
       "authors": ["Y. Liu", "G. Deng", "Y. Li", "K. Wang", "Z. Wang"],
       "year": 2023,
       "arxiv_id": "2306.05499",
       "relevance": "Introduces the HOUYI genetic attack framework used to evaluate defense robustness in this study."
     },
     {
       "title": "AutoDAN: Generating Stealthy Jailbreak Prompts on Aligned Large Language Models",
       "authors": ["X. Liu", "N. Xu", "M. Chen", "C. Xiao"],
       "year": 2023,
       "arxiv_id": "2310.04451",
       "relevance": "Adversarial attack method for generating stealthy jailbreak prompts against aligned LLMs."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["J. Achiam", "S. Adler", "S. Agarwal", "L. Ahmad"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, a major LLM whose security properties are relevant to the prompt injection defense landscape."
     },
     {
       "title": "GPTFuzzer: Red Teaming Large Language Models with Auto-Generated Jailbreak Prompts",
       "authors": ["J. Yu", "X. Lin", "Z. Yu", "X. Xing"],
       "year": 2024,
       "arxiv_id": "2309.10253",
       "relevance": "Automated red-teaming framework for testing LLM robustness against jailbreak attacks."
     },
     {
       "title": "LLaMA 2: Open Foundation and Fine-Tuned Chat Models",
       "authors": ["H. Touvron", "L. Martin", "K. Stone", "P. Albert"],
       "year": 2023,
       "relevance": "Open-source foundation model used as one of the JATMO fine-tuned base models in this study."
     },
     {
       "title": "Enhancing System Security: LLM-Driven Defense Against Prompt Injection Vulnerabilities",
       "authors": ["O. Muliarevych"],
       "year": 2024,
       "relevance": "Proposes middleware-based LLM validation layers for prompt injection defense, a competing approach to JATMO."
     },
     {
       "title": "Dynamic Moving Target Defense for Mitigating Targeted LLM Prompt Injection",
       "authors": ["S. Panterino", "M. Fellington"],
       "year": 2024,
       "relevance": "Proposes dynamic randomization-based defense against prompt injection, another defense strategy discussed in the related work."
     }
   ]
 }