ai-research-survey

scan.json (27771B)

---

 {
   "paper": {
     "title": "Early Approaches to Adversarial Fine-Tuning for Prompt Injection Defense: A 2022 Study of GPT-3 and Contemporary Models",
     "authors": [
       "Gustavo Sandoval",
       "Denys Fenchenko",
       "Junyao Chen"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.14271",
     "doi": "10.48550/arXiv.2509.14271"
   },
   "scan_version": 2,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Adversarial fine-tuning using structured <userInput> delimiters reduced prompt injection attack success rates from ~26-31% to near 0% for smaller GPT-3 models (Ada, Babbage, Curie). A positive correlation was found between model size and vulnerability to prompt injection, with Davinci (175B) being roughly four times as vulnerable as Ada (2.7B). The defense was not tested on the largest model (Davinci) due to cost, and a reinforcement learning variant failed due to compute constraints. The authors retrospectively acknowledge that fine-tuning-based defenses show poor generalization to novel attack patterns.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository is explicitly linked: https://github.com/GusSand/PromptInject. The paper references specific notebooks (dataset_construct.ipynb, original_openai.ipynb, openai_fine-tuned.ipynb, gpt-2_experiments.ipynb, non_openai_models.ipynb, Reinforcement_Learning-fine-tuning.ipynb)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The adversarial dataset construction is in the repo, fine-tuning datasets are from public Kaggle sources (Dave 2021, Kushare 2021, Shahane 2021, Vonteru 2019), and the paper states 'the reader can also find the logs from all attacks on the original language models and their fine-tuned versions in the results directory.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification mentioned in the paper. Only implicit references to Google Colab and OpenAI API."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While specific notebooks are named, there are no step-by-step reproduction instructions. A researcher would need to reverse-engineer the workflow from notebook names and the methodology description."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only point estimates (e.g., '26%', '31%', '0%'). No confidence intervals, error bars, or uncertainty measures are reported for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims fine-tuning reduces attacks and larger models are more vulnerable, comparing multiple models, but no statistical significance tests are used. Comparisons are based solely on comparing raw percentages."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 show attack success rates before and after fine-tuning with baseline context (e.g., Goal Hijacking on Ada: 26% before → 0% after; Babbage: 31% → 0%), allowing readers to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "They test 1,260 attack variations (35 prompts × 10 attack strings × some model parameters) but provide no justification for why this number is sufficient, no power analysis, and no discussion of whether the sample adequately covers the attack space."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures reported. Results appear to be single-run point estimates with no indication of result stability across repeated runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The undefended models serve as baselines (before fine-tuning). Table 1 shows before/after comparisons for Ada, Babbage, and Curie. Table 2 shows attack rates on additional models without fine-tuning."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "For the 2022 timeframe, GPT-3 variants (Ada, Babbage, Curie, Davinci), GPT-2, OPT-350M, and T-5 were contemporary models. The paper uses the PromptInject framework from Perez and Ribeiro (2022)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The adversarial fine-tuning approach has multiple components (structured delimiters, adversarial examples, fine-tuning), but no ablation isolates their individual contributions. For example, no test of delimiters without adversarial examples, or adversarial examples without delimiters."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two distinct attack categories are measured separately: goal hijacking success rate and prompt leaking success rate (Tables 1 and 2). Levenshtein distance-based similarity scoring is used with a threshold."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via Levenshtein distance similarity scores with a threshold. No human evaluation of attack success or defense quality is reported. Human evaluation would be relevant to assess borderline cases where the automated metric may misjudge attack success."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit separation between the fine-tuning data and the test attack data is described. The paper does not state whether the 1,260 attack variations used for testing overlap with the adversarial examples used in fine-tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model (Ada, Babbage, Curie, Davinci, GPT-2, OPT, T-5), by attack type (goal hijacking vs. prompt leaking), and by before/after fine-tuning (Tables 1 and 2)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative analysis of where attacks succeeded, what types of prompts were most effective, or specific examples of successful vs. failed attacks. The Babbage/Curie anomaly is noted but not analyzed in depth."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results reported: RL fine-tuning approach crashed on Google Colab Pro (couldn't complete training), Davinci could not be fine-tuned due to financial cost, and the 'Discovered Limitations' section acknowledges fine-tuning fragility and poor generalization."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims match results: '31% of the time' is supported by Table 1 (Babbage goal hijacking = 31%), 'reduced to near zero' is supported by Table 1 (all post-fine-tuning rates 0% except one 2.86% case), 'more flexible models exhibit greater vulnerability' is supported by Figure 5 and Tables 1-2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that adversarial fine-tuning reduces attack success is supported by a direct intervention design: same models tested before and after fine-tuning on the same attack set. This controlled comparison adequately supports the causal inference, though the lack of a held-out test set weakens it."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds its claims to the tested models and era: 'specific models tested are now superseded,' 'conducted in 2022,' 'Note: This methodology was developed for the 2022 model landscape.' The 'Discovered Limitations' section further acknowledges generalization gaps."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The 'Discovered Limitations' section discusses alternative explanations: fine-tuning may reduce rather than enhance safety, modern attacks bypass training-based defenses, the approach has poor generalization. The Babbage > Curie vulnerability anomaly is noted as an exception to the size-vulnerability trend."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Levenshtein distance similarity above a threshold as a proxy for 'attack success' but does not discuss the gap between this automated metric and actual vulnerability. Levenshtein distance may miss semantically successful attacks with low string similarity, or flag benign outputs with high similarity. The threshold value is not justified."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are listed: text-davinci-003, text-curie-001, text-babbage-001, text-ada-001 (with API-specific version suffixes). GPT-2 1.5B, OPT 350M, and T-5 small 60M are also specified with sizes."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Actual prompt text is provided. Figure 4 shows a complete prompt-completion pair with the <userInput> delimiter approach. Figure 2 shows attack examples. The paper includes examples like 'Correct this to standard English: <userInput>...'."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is mentioned as a parameter that 'define the confidence level with which the model is making its predictions' but specific values are not reported. No table or section lists the actual hyperparameter settings used (temperature, top-p, max tokens, etc.)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is direct prompt-based testing and fine-tuning via APIs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The dataset construction process is documented: 35 base prompts, 5 attack variations per category (goal hijacking and prompt leaking), 1,260 total variations. Fine-tuning datasets sourced from Kaggle, augmented with <userInput> tags, and formatted into JSONL for OpenAI's API."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Dedicated 'Discovered Limitations' and 'Contemporary Relevance and Limitations' sections with substantive discussion of fine-tuning fragility, attack evolution, and generalization gaps."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: '2024 studies show that fine-tuning can inadvertently reduce safety alignment, even on benign datasets,' 'adversarial fine-tuning shows poor generalization to novel attack patterns,' and 'modern attacks like many-shot jailbreaking and indirect injection bypass training-based defenses.' These are specific to this study's approach."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries: 'specific models tested are now superseded,' the work is 'a starting point for more sophisticated defense mechanisms, rather than a complete solution,' 'this methodology was developed for the 2022 model landscape,' and they list specific things they did NOT test (Davinci fine-tuning, modern architectures)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'the reader can also find the logs from all attacks on the original language models and their fine-tuned versions in the results directory' in the GitHub repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: adversarial dataset built from 35 base prompts × attack variations using the PromptInject framework; fine-tuning datasets from specific Kaggle sources (Dave 2021 for translation, Kushare 2021 for grammar, Shahane 2021 for sentiment, Vonteru 2019 for summarization)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public datasets from Kaggle and procedurally generated adversarial prompts."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: base prompts → attack string injection → 1,260 attack variations → model testing via API → Levenshtein distance scoring → threshold-based success determination. Fine-tuning pipeline: Kaggle datasets → tag augmentation → JSONL formatting → OpenAI fine-tuning API."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed. While the authors appear to be NYU students (based on @nyu.edu emails), no funding source or acknowledgment of unfunded status is stated."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are provided via NYU email addresses ({gs157, df1911, jc9723}@nyu.edu). They are not evaluating a product from their own employer."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "This appears to be unfunded student work at NYU (based on @nyu.edu emails and no acknowledgments section). No funder to assess independence of."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest declaration statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests defenses against adversarial prompt injection attacks rather than evaluating model knowledge on a benchmark. The attacks are procedurally generated at test time, so training data contamination of benchmark answers is not the relevant concern."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests adversarial defenses with procedurally generated attack prompts, not pre-trained model knowledge on fixed benchmarks. Standard contamination concerns do not apply."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No fixed benchmark is used that could have been in training data. The adversarial attacks are constructed specifically for this study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or wall-clock times reported. The paper mentions Davinci fine-tuning was too expensive and RL training exceeded Colab Pro RAM, but no actual cost figures are given."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget stated. Google Colab Pro is mentioned as insufficient for RL training, but no GPU hours, API spend, or hardware specifications are quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be single-run measurements."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. The 1,260 variations are the test dataset size, not repeated runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature and other parameters are mentioned as configurable but no search budget, number of configurations tried, or search method is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the final configuration was selected or whether the reported results represent a best-case selection from multiple attempts."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so there are no p-values to correct for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own adversarial fine-tuning defense against undefended models without acknowledging self-evaluation bias. No independent evaluation or re-implementation by others is mentioned."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of vastly different sizes are compared (60M T-5 to 175B Davinci) without normalizing for compute cost. The fine-tuning was only applied to smaller models due to cost, but no compute-performance tradeoff analysis is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "Attack success is measured via Levenshtein distance similarity threshold, but the paper does not discuss whether this metric adequately captures real-world prompt injection vulnerability. The threshold value is not justified, and there is no analysis of false positives/negatives in the automated scoring."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in the study. Models are tested directly via API calls."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the tested models may have seen similar prompt injection patterns during pre-training, despite PromptInject framework content potentially being in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information about expected behavior through the prompt structure."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 1,260 attack variations are independent or structurally correlated (e.g., variations from the same base prompt may not be independent tests)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is described."
       }
     }
   },
   "claims": [
     {
       "claim": "Without adversarial fine-tuning, prompt injection attacks succeeded 31% of the time on GPT-3 series models (Babbage goal hijacking)",
       "evidence": "Table 1 shows Babbage goal hijacking at 31%, Ada at 26%, Curie at 18% before fine-tuning. Table 2 shows Davinci at 24.28%.",
       "supported": "strong"
     },
     {
       "claim": "Adversarial fine-tuning reduced attack success rates to near zero for Ada, Babbage, and Curie models",
       "evidence": "Table 1 shows all goal hijacking rates drop to 0% post-fine-tuning. Prompt leaking remains at 2.86% for Ada only, all others 0%.",
       "supported": "strong"
     },
     {
       "claim": "There is a positive correlation between model size and vulnerability to prompt injection attacks",
       "evidence": "Figure 5 shows increasing attack success rate with model size. Davinci (175B) at ~24% vs Ada (2.7B) at ~14%. However, Babbage (6.7B) appears more vulnerable than Curie (13B), which the authors note as an anomaly.",
       "supported": "moderate"
     },
     {
       "claim": "The structured delimiter approach (<userInput> tags) anticipated modern instruction hierarchy systems",
       "evidence": "The paper draws parallels to OpenAI's instruction hierarchy (Wallace et al. 2024) and Anthropic's Constitutional AI, but provides no evidence of direct influence. This is retrospective framing added in 2025.",
       "supported": "weak"
     },
     {
       "claim": "The same prompt injection vulnerabilities exist in non-GPT-3 models (GPT-2, OPT, T-5)",
       "evidence": "Table 2 shows goal hijacking success rates for GPT-2 (7.85%), OPT (45.71%), and T-5 (8.57%), confirming vulnerability across architectures.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Retrospective overclaiming of influence",
       "detail": "The paper was conducted in 2022 but published in 2025 with extensive retrospective additions claiming influence on OpenAI's instruction hierarchy, Anthropic's Constitutional AI, and other modern defenses. No evidence of direct influence is provided. These sections appear to inflate the paper's significance."
     },
     {
       "flag": "No statistical rigor",
       "detail": "All results are point estimates without confidence intervals, error bars, significance tests, or multi-run variance. Comparisons between models and between before/after fine-tuning lack any statistical foundation."
     },
     {
       "flag": "Incomplete evaluation on the most important model",
       "detail": "Davinci (175B), the most important and most vulnerable GPT-3 model, was not tested with the adversarial fine-tuning defense due to cost. The defense is only validated on the three smallest models, leaving the key question unanswered."
     },
     {
       "flag": "Unclear train-test separation",
       "detail": "The paper does not explain whether the adversarial examples used in fine-tuning overlap with the attack prompts used in evaluation. If the model was fine-tuned on the same attacks it was tested on, the near-zero results would be trivially expected."
     },
     {
       "flag": "Automated metric not validated",
       "detail": "Attack success is determined by Levenshtein distance similarity above an unspecified threshold. This automated metric is not validated against human judgment of whether attacks actually succeeded."
     }
   ],
   "cited_papers": [
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": ["Tom Brown"],
       "year": 2020,
       "relevance": "Foundational GPT-3 paper establishing the large language model paradigm that prompt injection attacks target."
     },
     {
       "title": "Ignore Previous Prompt: Attack Techniques For Language Models",
       "authors": ["Fábio Perez", "Ian Ribeiro"],
       "year": 2022,
       "relevance": "Defines the PromptInject framework for goal hijacking and prompt leaking attacks, which this paper directly builds upon."
     },
     {
       "title": "Language Models are Unsupervised Multitask Learners",
       "authors": ["Alec Radford", "Jeffrey Wu", "Rewon Child", "David Luan", "Dario Amodei", "Ilya Sutskever"],
       "year": 2019,
       "relevance": "GPT-2 paper establishing the decoder-only transformer architecture evaluated in this study."
     },
     {
       "title": "The Instruction Hierarchy: Training LLMs to Prioritize Privileged Instructions",
       "authors": ["Eric Wallace"],
       "year": 2024,
       "arxiv_id": "2404.13208",
       "relevance": "Modern prompt injection defense that formalizes the delimiter-based separation approach explored in this paper."
     },
     {
       "title": "SecAlign: Defending Against Prompt Injection with Preference Optimization",
       "authors": ["Sizhe Wang"],
       "year": 2024,
       "arxiv_id": "2410.05451",
       "relevance": "Addresses generalization limitations of adversarial fine-tuning defenses through preference optimization."
     },
     {
       "title": "Constitutional Classifiers: Defending against universal jailbreaks",
       "authors": ["Anthropic"],
       "year": 2024,
       "relevance": "Extends adversarial training concepts with preference learning for LLM safety alignment."
     },
     {
       "title": "Many-shot jailbreaking",
       "authors": ["Anthropic"],
       "year": 2024,
       "relevance": "Documents modern prompt injection attack that bypasses training-based defenses, illustrating limitations of the fine-tuning approach."
     },
     {
       "title": "Generating Textual Adversarial Examples for Deep Learning Models: A Survey",
       "authors": ["Wei Emma Zhang", "Quan Z. Sheng", "Ahoud Alhazmi"],
       "year": 2019,
       "relevance": "Survey of adversarial attacks on NLP models providing background on defense strategies including adversarial training."
     },
     {
       "title": "Pre-train, Prompt, and Predict: A Systematic Survey of Prompting Methods in Natural Language Processing",
       "authors": ["Pengfei Liu", "Weizhe Yuan", "Jinlan Fu", "Zhengbao Jiang", "Hiroaki Hayashi", "Graham Neubig"],
       "year": 2021,
       "relevance": "Comprehensive survey of prompting methods providing the prompt engineering framework that prompt injection attacks exploit."
     }
   ]
 }