ai-research-survey

scan.json (29143B)

---

 {
   "paper": {
     "title": "Prompt Injection Attacks in Defended Systems",
     "authors": ["Daniil Khomsky", "Narek Maloyan", "Bulat Nutfullin"],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2406.14048",
     "doi": "10.1007/978-3-031-80853-1_30"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "The paper investigates prompt injection attacks and defenses in the SaTML 2024 CTF competition, achieving 8th in attack and 9th in defense. Code-based attacks (e.g., asking the model to execute scripts or populate data structures with its secret) were the most effective category, particularly against GPT-3.5. The paper demonstrates that combining multiple basic attack strategies (distraction, teacher, system prompt, word-splitting, code-based) can bypass three-tiered defenses (system prompt, Python filter, LLM filter). GPT-3.5 was more susceptible to complex encoded attacks while Llama-2 leaked more system prompt information but struggled with encryption tasks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper. The Python filter code is shown inline but no reproducible codebase is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper references the competition dataset released by organizers at https://huggingface.co/datasets/ethz-spylab/ctf-satml24 (Section III-F). This publicly available dataset contains the attacks, defenses, and secrets used by participants."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, library versions, or setup instructions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. The paper describes attack and defense strategies but provides no guide to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Section V claims 'Confidence intervals and significance tests were used to assess the robustness of the results' but no actual confidence intervals or error bars appear in the tables or text. Tables I and II show only point estimates."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Section V claims significance tests were used but none are actually shown — no p-values, test statistics, or test names appear anywhere in the paper."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw competition scores are reported (e.g., total attack score 3428 vs best team 9125). No formal effect sizes, relative improvements with context, or standardized measures are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample size. The number of defenses attacked, total number of competing teams, or power considerations are not discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures reported. All results are single-value competition scores."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Tables I and II compare the authors' results against the top 3 teams for both defense and attack, providing competitive baselines from the competition."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are other teams from the same SaTML 2024 CTF competition, making them contemporary by definition."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The paper describes multiple attack categories (distraction, teacher, system prompt, word-splitting, code-based) but never systematically measures the contribution of individual components or combinations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both defense metrics (vD normalization coefficient, Table I) and attack metrics (cumulative scoreD, Table II), and separates results by model (Llama-2 vs GPT-3.5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of attack quality or defense effectiveness is performed. All evaluation is through the automated competition scoring system."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The competition structure separates the Reconnaissance sub-phase (exploration) from the Evaluation sub-phase (actual attack attempts with scores recorded), as described in Section III. This provides a structural separation between development and evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model (Llama-2 vs GPT-3.5) in Tables I and II. Attack methods are categorized into distinct groups (distraction, teacher, system prompt, word-splitting, code-based, combined) with qualitative analysis per category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where attacks failed: e.g., GPT-3.5 does not output its system prompt upon direct request (Section IV-A-3), Llama-2 cannot handle ASCII encoding or decryption tasks (Section IV-C), and certain code-based attacks fail against Python filters that detect secrets in output (Section IV-B)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: the LLM filter was not used because it failed utility metrics (Section IV-B), ASCII code attacks fail when the model explains its work alongside the output (Section IV-C), and the initial defense had 'obvious flaws' discovered during the attack phase (Section IV-B)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claims are modest: it states the paper 'investigates methods for black-box attacks' and 'presents a methodology for vulnerability detection and defensive strategies.' The paper delivers on these through the CTF participation and detailed attack/defense descriptions."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims such as 'the Python filter helps against attacks' and certain attack combinations 'successfully retrieved secret values' without controlled experimental design to isolate causal factors. The competition setting doesn't control for confounds between attack components."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Prompt Injection Attacks in Defended Systems' implies generality beyond what was tested. Results are specific to GPT-3.5 and Llama-2 in the SaTML 2024 CTF with 512-character-limited defenses and 6-character random secrets, but the paper does not bound its claims to these specific conditions."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for why certain attacks succeed or fail. For example, the paper doesn't consider whether attack success depends on model size, training data, or RLHF alignment strength rather than the attack strategy itself."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures secret extraction success, which directly corresponds to its claim about attack effectiveness. The CTF scoring formula (Section III-G) explicitly defines what success means, and the paper doesn't overclaim beyond this measurement."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'Chat-GPT 3.5' and 'Llama 2' without specific versions, snapshot dates, or API versions (e.g., no 'gpt-3.5-turbo-0613' or 'Llama-2-7b-chat')."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Extensive actual prompt text is provided throughout: the system defense prompt (Section III-H), the full Python filter code (Section IV-B), and numerous actual attack prompts with both user inputs and model responses (Sections IV-A and IV-C)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported — no temperature, top-p, max tokens, or other generation settings for either GPT-3.5 or Llama-2."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-tiered defense architecture (system prompt, Python filter, LLM filter) is described in detail in Section III, including filter ordering, character limits, input/output format, and the Debug defense mode. Figure 1 shows the architecture diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data preprocessing steps are documented. The paper does not describe how competition data was processed for analysis or how attack/defense pairs were selected for reporting."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. The conclusion contains only a brief forward-looking statement about 'ongoing need to enhance AI security' without substantive self-criticism."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential issues such as the artificial nature of the CTF setting, the limited model selection, or the non-representativeness of 6-character random secrets."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what its results do NOT show — e.g., whether findings transfer to production systems, other models, longer secrets, or non-English attacks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The competition dataset containing attacks and defenses was released by the organizers at huggingface.co/datasets/ethz-spylab/ctf-satml24 (Section III-F), allowing independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III describes the CTF competition structure in detail: two phases (Defense and Attack), the Attack phase split into Reconnaissance and Evaluation, the scoring formula, and how secrets were generated (random alphanumeric sequence of 6 characters)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "No description of how CTF participants were recruited, how many teams participated in total, or whether the participant pool introduces selection bias (e.g., only security-focused researchers)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of the pipeline from raw competition interactions to the analyzed results presented in the paper. The paper does not explain how specific examples were selected or how the tables were compiled from the competition data."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. There is no acknowledgments section mentioning grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "Only a personal email address (homdanil123@gmail.com) is provided. No institutional affiliations are listed for any of the three authors."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed at all, making it impossible to assess funder independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests attack and defense strategies against LLMs in a CTF competition, not model knowledge on benchmarks. The secrets are randomly generated per defense, so training data contamination is not a concern."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests defense/attack strategies, not model capability on benchmarks. Secrets are freshly generated random strings, so train/test overlap is structurally impossible."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The CTF uses randomly generated secrets and custom defenses, not pre-existing benchmarks. Contamination in the traditional sense does not apply to this evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The paper reports results from a CTF competition where participants are adversarial researchers, not study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study — the paper analyzes attack/defense strategies in a competition setting."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Competition participant demographics are not relevant as human subjects data."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experimental study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experimental study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section VI states the approach is 'cheap in terms of resources spent' because it 'does not involve model training' but provides no quantification of API costs, tokens consumed, or wall-clock time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated — no API costs, GPU hours, or total resource consumption for the attack or defense experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. Results are from a single competition run with no repeated experiments across different random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. While the scoring formula includes X (number of chats created), the paper does not report how many attempts were made per defense."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper does not describe how attack prompts or defense configurations were tuned or how many variations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why the final attack/defense configurations were selected. The paper presents the chosen strategies without describing the selection process or reporting alternative configurations tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are actually shown despite being claimed in Section V, so no multiple comparison correction could have been applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own attack and defense methods in the competition but do not acknowledge the bias of self-evaluation or discuss how this might affect their interpretation of results."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of performance as a function of compute budget. The scoring formula penalizes more attempts (chats) but the paper does not analyze how resource expenditure relates to attack success."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the CTF competition format (protecting 6-character random secrets with 512-character-limited defenses) actually measures real-world LLM security properties. The ecological validity of the benchmark is not questioned."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The defense scaffolding (system prompt + Python filter + LLM filter) IS the thing being tested. The paper evaluates bundled defense systems, not isolated model capabilities, so the scaffold confound does not apply."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. While the CTF uses random secrets (mitigating direct leakage), the paper does not discuss whether models may have seen similar attack patterns or defense strategies in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The paper does not consider whether the competition setup leaks information (e.g., the system prompt structure being known to attackers provides privileged information)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of non-independence. Multiple attacks against the same defense are not independent, and the paper does not address whether correlation between defense strategies affects the analysis."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is described or applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Despite advanced three-tiered defenses (system prompt, Python filter, LLM filter), significant security gaps remain exploitable through prompt injection attacks on LLMs.",
       "evidence": "Competition results showing the authors' defense ranked 9th (Table I) with multiple teams breaking it, and their attacks ranked 8th (Table II) demonstrating they could break many defenses. Section IV-C shows specific combined attacks that bypass all three defense layers.",
       "supported": "moderate"
     },
     {
       "claim": "Code-based attacks are the most promising attack category against defended LLMs.",
       "evidence": "Section IV-A-5 describes code-based attacks as 'the largest and most promising group' with multiple working examples (Python variable initialization, JSON object creation, script execution). Section IV-C shows the code-based attack with ASCII encoding is 'one of the most successful' combined approaches.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-3.5 is more susceptible to complex encoded attacks while Llama-2 provides more information about system prompts but cannot handle complex encryption tasks.",
       "evidence": "Section IV-C states 'the GPT-3.5 model is much higher quality than the Llama-2 model' for attack purposes, noting Llama-2 'could not cope with ASCII encoding or other encryption and decryption tasks.' Section IV-A-3 shows Llama-2 outputs its system prompt on direct request while GPT-3.5 does not.",
       "supported": "moderate"
     },
     {
       "claim": "Confidence intervals and significance tests confirmed the effectiveness of the proposed methods.",
       "evidence": "Section V states 'Confidence intervals and significance tests were used to assess the robustness of the results. The detailed statistical analyses confirmed the effectiveness' but no actual statistical results, p-values, or confidence intervals are shown anywhere in the paper.",
       "supported": "unsupported"
     },
     {
       "claim": "The attack and defense architecture is resource-efficient because it does not require model training or large computing resources.",
       "evidence": "Section VI states the approach is 'cheap in terms of resources spent, because it does not involve model training, the presence of a large amount of marked data and large computing resources.' No quantitative cost data is provided.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Phantom statistical analyses",
       "detail": "Section V explicitly claims 'Confidence intervals and significance tests were used to assess the robustness of the results' but no actual statistical results appear anywhere in the paper — no p-values, no confidence intervals, no test statistics. This is a serious credibility concern."
     },
     {
       "flag": "Missing competition context",
       "detail": "The paper reports ranking 8th in attack and 9th in defense but never states the total number of participating teams, making it impossible to assess whether these rankings represent strong or weak performance."
     },
     {
       "flag": "No model version specificity",
       "detail": "Only 'Chat-GPT 3.5' and 'Llama 2' are specified without versions, sizes, or API dates. Model behavior can vary significantly across versions."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any formal discussion of limitations, threats to validity, or scope boundaries despite making broad claims about LLM security."
     },
     {
       "flag": "Missing author affiliations",
       "detail": "Only a personal Gmail address is provided. No institutional affiliations are disclosed for any of the three authors."
     },
     {
       "flag": "Qualitative rather than quantitative attack comparison",
       "detail": "Attack categories (distraction, teacher, system prompt, word-splitting, code-based) are described with examples but never quantitatively compared — no success rates per category, no breakdown of which attacks worked against which defenses."
     }
   ],
   "cited_papers": [
     {
       "title": "Jailbreak and Guard Aligned Language Models with Only Few In-Context Demonstrations",
       "authors": ["Zeming Wei", "Yifei Wang", "Yisen Wang"],
       "year": 2023,
       "arxiv_id": "2310.06387",
       "relevance": "Introduces contextual attack and defense concepts for LLM jailbreaking, directly relevant to prompt injection security research."
     },
     {
       "title": "Multi-step Jailbreaking Privacy Attacks on ChatGPT",
       "authors": ["Haoran Li", "Dadi Guo", "Wei Fan", "Mingshi Xu", "Jie Huang", "Fanpu Meng", "Yangqiu Song"],
       "year": 2023,
       "arxiv_id": "2304.05197",
       "relevance": "Investigates multi-step privacy attacks on ChatGPT, directly relevant to LLM security and prompt injection research."
     },
     {
       "title": "Jailbreaking ChatGPT via Prompt Engineering: An Empirical Study",
       "authors": ["Yi Liu", "Gelei Deng", "Zhengzi Xu"],
       "year": 2024,
       "arxiv_id": "2305.13860",
       "relevance": "Empirical study of prompt-based jailbreaking methods, core reference for LLM security evaluation methodology."
     },
     {
       "title": "Defending ChatGPT against Jailbreak Attack via Self-Reminder",
       "authors": ["Fangzhao Wu", "Yueqi Xie", "Jingwei Yi"],
       "year": 2023,
       "relevance": "Proposes self-reminder defense mechanism against jailbreaks, directly relevant to LLM defense strategies."
     },
     {
       "title": "Bergeron: Combating Adversarial Attacks through a Conscience-Based Alignment Framework",
       "authors": ["Matthew Pisano", "Peter Ly", "Abraham Sanders"],
       "year": 2024,
       "arxiv_id": "2312.00029",
       "relevance": "Proposes auxiliary LLM defense framework against adversarial attacks, relevant to multi-tier LLM defense architectures."
     },
     {
       "title": "Certifying LLM Safety against Adversarial Prompting",
       "authors": ["Aounon Kumar", "Chirag Agarwal", "Suraj Srinivas"],
       "year": 2024,
       "arxiv_id": "2309.02705",
       "relevance": "Proposes certified defenses against adversarial prompts using input permutation, relevant to prompt injection defense research."
     },
     {
       "title": "Low-Resource Languages Jailbreak GPT-4",
       "authors": ["Zheng-Xin Yong", "Cristina Menghini", "Stephen H. Bach"],
       "year": 2023,
       "arxiv_id": "2310.02446",
       "relevance": "Demonstrates jailbreaking via translation to low-resource languages, raising safety concerns for multilingual LLM deployment."
     },
     {
       "title": "Universal and Transferable Adversarial Attacks on Aligned Language Models",
       "authors": ["Andy Zou", "Zifan Wang", "J. Zico Kolter", "Matt Fredrikson"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Introduces the AdvBenchmark dataset and universal adversarial attack methods for aligned LLMs, foundational work in LLM adversarial robustness."
     },
     {
       "title": "Jailbroken: How Does LLM Safety Training Fail?",
       "authors": ["Alexander Wei", "Nika Haghtalab", "Jacob Steinhardt"],
       "year": 2023,
       "relevance": "Analyzes failure modes of LLM safety training including base64 encoding attacks, directly relevant to understanding prompt injection vulnerabilities."
     },
     {
       "title": "Defending Against Alignment-Breaking Attacks via Robustly Aligned LLM",
       "authors": ["Bochuan Cao", "Yuanpu Cao", "Lu Lin", "Jinghui Chen"],
       "year": 2023,
       "arxiv_id": "2309.14348",
       "relevance": "Proposes robust alignment defense against jailbreak attacks, relevant to LLM defense methodology."
     },
     {
       "title": "SmoothLLM: Defending Large Language Models Against Jailbreaking Attacks",
       "authors": ["Alexander Robey", "Eric Wong", "Hamed Hassani", "George J. Pappas"],
       "year": 2023,
       "arxiv_id": "2310.03684",
       "relevance": "Proposes input perturbation defense against jailbreaking, relevant to practical LLM defense strategies."
     },
     {
       "title": "Survey of Hallucination in Natural Language Generation",
       "authors": ["Ziwei Ji", "Nayeon Lee", "Rita Frieske"],
       "year": 2023,
       "relevance": "Comprehensive survey of LLM hallucination, relevant to understanding LLM reliability and failure modes."
     }
   ]
 }