ai-research-survey

scan.json (27987B)

---

 {
   "paper": {
     "title": "AEGIS: Automated Co-Evolutionary Framework for Guarding Prompt Injection",
     "authors": [
       "Ting-Chun Liu",
       "Ching-Yu Hsu",
       "Kuan-Yi Lee",
       "Chi-An Fu",
       "Hung-yi Lee"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2509.00088"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link, Zenodo archive, or other repository URL is provided in the paper. No mention of code release in abstract, footnotes, or anywhere in the text."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The dataset consists of student-submitted articles from a National Taiwan University course. The paper states they were 'manually modified to anonymize personal data and for copyright purposes' but no download link or public release of the data is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists the LLM used (gpt-4.1-mini) in the hyperparameters table (Appendix A.3), but does not provide library versions, a requirements.txt, Dockerfile, or other environment specification sufficient to reproduce the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No README, step-by-step commands, or reproduction scripts are referenced. The algorithms (Algorithms 1 and 2) describe the procedure at a high level, but are not runnable reproduction instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 states experiments were conducted three times and standard deviation was calculated. Figures 4 and 5 show shaded regions representing standard deviation across runs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests (t-tests, p-values, bootstrap, etc.) are applied when comparing AEGIS to baselines. Claims of outperformance are based solely on comparing point estimates (e.g., TPR 0.84 vs. 0.64) without any significance testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract explicitly states the defense improves TPR by 0.20 compared to the previous state of the art (from 0.64 to 0.84), with only a slight decrease in TNR of 0.02. Table 1 provides the baseline context needed to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The training set uses 50 GPT-generated benign articles (40 train, 10 validation), and evaluation uses 143 malicious + 100 benign student articles. No power analysis or justification for these sample sizes is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 states: 'all experiments were conducted three times. The results presented in this paper are the average values from these three runs. We also calculated the standard deviation.' Figures 4 and 5 show shaded standard deviation regions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares AEGIS against three baselines: Perplexity-based Detection (Alon and Kamfonas, 2023), LLaMA 3.1 Guard (Inan et al., 2023), and the Human-Crafted Prompt from Chiang et al. (2024)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are from 2023-2025 and include LLaMA Guard (2023) and recently published prompt injection defenses (PromptArmor 2025, MELON 2025). The Human-Crafted Prompt (Chiang et al., 2024) is the direct predecessor on the same dataset."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 presents ablation studies removing the gradient buffer (Section 6.1), removing multiple gradients (Section 6.2), and training only one side of the GAN (Section 6.3). Figures 4 and 5 show the performance differences across ablation conditions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Both True Positive Rate (TPR) and True Negative Rate (TNR) are reported for defense evaluation, and Attack Success Rate (ASR) and relative score change (delta_Srel) are reported for attacker evaluation."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper's claims are about automated prompt injection detection performance on a pre-labeled dataset. Human evaluation of the system's outputs is not relevant to the core claims; TPR/TNR against pre-labeled attack/benign articles is the appropriate evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 clearly distinguishes training data (50 GPT-generated benign articles, 40 train / 10 validation) from the real-world evaluation set (143 malicious student articles + 100 benign articles from a prior course). The evaluation set was not used during training."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "The 143 malicious articles contain 'a wide variety of successful prompt injections' with different strategies. A per-category breakdown by injection type would be meaningful but is not provided. Only aggregate TPR and TNR are reported, hiding potential variation across injection strategies."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not provide error analysis or qualitative examples of failure cases. Appendix A.4 shows defense prompt improvement examples (successes), but no cases where the defense fails to detect injections are discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study in Section 6 reports components that degrade performance when removed: removing the gradient buffer leads to ~5% TPR degradation and slower convergence; removing multiple gradients leads to over 10% performance degradation."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims TPR improvement of 0.20 over the previous state of the art with only 0.02 decrease in TNR. Table 1 confirms: AEGIS Iteration 8 achieves TPR=0.84 vs. Human-Crafted Prompt TPR=0.64 (difference 0.20), and TNR=0.89 vs. 0.91 (difference -0.02). The claim about outperforming existing baselines is confirmed by Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies (Section 6) use controlled single-variable manipulation to support causal claims about component contributions. The co-evolution, gradient buffer, and multi-route gradients are each removed individually, providing adequate causal evidence for their contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper is evaluated on a single task (automated assignment grading at NTU) and the Limitation section acknowledges this ('our evaluation focused on automated assignment grading task, which may not fully capture the diversity of real-world security-sensitive tasks'). However, the abstract and title claim general applicability ('Automated Co-Evolutionary Framework for Guarding Prompt Injection') without consistently bounding the scope to the specific tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why AEGIS outperforms baselines. No consideration is given to whether improvements might be due to dataset-specific factors (e.g., the student article data being from the same institution as the defense prompt development), or to the model (gpt-4.1-mini) being particularly well-suited to the evaluation setup."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'gpt-4.1-mini', 'GPT-5-mini', 'GPT-4.1-nano', 'Gemini-2.0-flash', 'Gemini-2.5-flash-lite'. Per the schema: 'Marketing names like \"Gemini-2.5\" or \"GPT-4o\" without a snapshot date or API version do NOT count as specified versions.' No snapshot dates or API version identifiers are provided for any model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.4 (Table 6) shows actual full prompt text for the defense prompts at GAN iterations 0, 4, and 8. Appendix A.6 (Table 11) provides the full Human-Crafted Prompt baseline text. The actual prompts sent to the model are shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 (Appendix A.3) lists framework-level hyperparameters (GAN iterations, optimization iterations, weights, power parameters) but does NOT include LLM API settings: temperature, top-p, max tokens, or sampling parameters. The schema states: 'If the paper uses an LLM API without stating temperature/sampling settings, NO.' The paper calls GPT and Gemini APIs extensively without reporting these settings."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic co-evolutionary scaffolding is described in detail in Section 3 and Appendix A.1-A.2, including Algorithm 1 (co-evolution framework), Algorithm 2 (TGO+ workflow), the evaluation functions Eval() and Val(), the gradient acquisition and application steps, and the gradient buffer mechanism."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes that the 143 malicious student articles were 'manually modified to anonymize personal data and for copyright purposes, while preserving their original strategic intent.' The training/validation split (40/10) and evaluation set composition (143 malicious + 100 benign) are clearly stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The paper has a dedicated 'Limitation' section (after Section 7/Conclusion) with substantive discussion of three specific limitations: single-task evaluation scope, text-only system focus, and lack of large-scale human evaluation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitation section provides specific threats: (1) evaluation limited to automated assignment grading task, (2) defense targets only text-based dialogue systems with effectiveness in multimodal settings unclear, (3) no large-scale human evaluation conducted. These are specific to the study rather than generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitation section explicitly states what was NOT shown: generalizability to diverse real-world security tasks is not demonstrated, effectiveness in multimodal systems is not tested, and large-scale human evaluation was not conducted."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The student-submitted articles used for evaluation are not publicly released. Section 4.1 mentions they were modified for anonymization and copyright purposes, but no download link or data access mechanism is provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the data sources: 50 GPT-generated benign articles for training/validation; 143 malicious articles and 100 benign articles from student submissions in a prior NTU course (same course as Chiang et al., 2024). The selection criteria are stated: malicious articles are ones that 'achieve full scores without being detected by the defense.'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "The data consists of course submissions from an existing course, not recruited participants. This is archival data from a prior course, not a human subjects study requiring participant recruitment description."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents the full data pipeline: GPT-generated benign articles for training (40) and validation (10); real-world evaluation uses 143 malicious student articles (those bypassing existing defense) + 100 benign student articles. The transformation step (anonymization/modification) is mentioned."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "There is no acknowledgments section and no mention of funding sources anywhere in the paper. It is unclear whether this is unfunded work or whether funding was simply not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Electrical Engineering at National Taiwan University, as stated on the first page with institutional email addresses. No conflict of interest exists with LLM providers; the paper evaluates GPT and Gemini models from third parties."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed; cannot assess funder independence. Marked as not applicable since funding source is absent from the paper."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests a defense framework/tool against prompt injection attacks, not evaluating a pre-trained model's capability on a benchmark. Per the schema: NA for 'studies that test defenses/tools rather than model knowledge.' The LLMs are used as tools within the defense system."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning: the paper tests a defense tool, not model knowledge on a benchmark. Contamination concerns about whether the model memorized benchmark data are not the primary concern here."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning: the paper evaluates a defense framework, not a model's benchmark performance. The schema specifies NA for papers that test defenses/tools rather than model knowledge."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not involve human participants in its experiments. The student-submitted data is archival course data, not a prospective human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses archival student course data that was already collected and anonymized. No prospective human subject enrollment is conducted, so IRB approval is not required (though the use of student data may warrant mention of ethics review)."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the experiments."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are recruited; the data consists of archival course submissions."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; no randomization to experimental conditions is needed."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or human evaluators of system outputs are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the experiments."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The framework calls GPT-4.1-mini and other OpenAI/Gemini APIs repeatedly across 8 GAN iterations with multiple optimization iterations each, but no API costs, token usage, or wall-clock time is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, total API spend, or hardware specifications are provided for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "AEGIS improves the true positive rate (TPR) by 0.20 compared to the previous state of the art, with only a slight decrease in the true negative rate (TNR) of 0.02.",
       "evidence": "Table 1 (Section 5.1): AEGIS at Iteration 8 achieves TPR=0.84 vs. Human-Crafted Prompt TPR=0.64 (difference 0.20) and TNR=0.89 vs. 0.91 (difference -0.02). Three-run average with standard deviation reported.",
       "supported": "strong"
     },
     {
       "claim": "Co-evolution is critical: training only one side of the GAN leads to overfitting to a static opponent and lower overall TPR.",
       "evidence": "Section 6.3 ablation study: 'The lack of an adaptive adversary meant that the trained model quickly overfit to its static opponent.' Figure 5 shows single-sided training achieves lower final TPR than the default method.",
       "supported": "moderate"
     },
     {
       "claim": "The gradient buffer improves convergence and TPR by approximately 5%.",
       "evidence": "Section 6.1: 'This change leads to a slower convergence rate and degrades the TPR in defense about 5%.' Figure 5 shows ablation without gradient buffer achieving lower TPR.",
       "supported": "moderate"
     },
     {
       "claim": "The AEGIS framework generalizes across multiple LLMs, with all tested models achieving better TPR at higher GAN iterations.",
       "evidence": "Table 3 (Section 5.3.1): GPT-5-mini, GPT-4.1-nano, Gemini-2.0-flash, and Gemini-2.5-flash-lite all show increasing TPR from GAN iteration 0 to 4. Table 4 (Section 5.3.2) shows transfer of GPT-4.1-mini-generated prompts to other models also improves with more iterations.",
       "supported": "moderate"
     },
     {
       "claim": "Defense prompts generated by AEGIS improve qualitatively over GAN iterations, becoming more specific and comprehensive.",
       "evidence": "Appendix A.4 (Table 6) shows the actual defense prompt text at iterations 0, 4, and 8, demonstrating increasing specificity in detecting subtle manipulation strategies. The iteration 0 prompt is short and general; iteration 8 covers indirect appeals, conditional statements, and specific manipulation patterns.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AEGIS is an automated co-evolutionary framework that jointly optimizes attack and defense prompts against prompt injection attacks using a GAN-inspired iterative process with textual gradient optimization (TGO+). On a real-world dataset of student-submitted articles with prompt injections from a National Taiwan University course, AEGIS achieves TPR=0.84 and TNR=0.89, outperforming the best baseline (Human-Crafted Prompt, TPR=0.64, TNR=0.91) by 0.20 TPR at the cost of 0.02 TNR. Ablation studies confirm that co-evolution, gradient buffering, and multi-objective optimization each contribute to performance. The framework generalizes across GPT and Gemini model families without model fine-tuning.",
   "red_flags": [
     {
       "flag": "Single narrow task generalization",
       "detail": "All evaluation is conducted on a single specific task (automated assignment grading at NTU) with data from a single course. The paper frames this as a general prompt injection defense framework but tests only one domain. The Limitation section acknowledges this but the abstract and introduction make broad claims about prompt injection defense generally."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Comparisons between AEGIS and baselines are made by comparing point estimates in Table 1 (e.g., TPR 0.84 vs. 0.64) without any statistical significance testing. With a test set of 243 articles (143 malicious + 100 benign) and three experimental runs, statistical testing would be feasible and warranted."
     },
     {
       "flag": "No inference cost reporting",
       "detail": "The framework calls large LLM APIs (GPT-4.1-mini, GPT-4o, Gemini models) across 8 GAN iterations with multiple optimization iterations each. No API costs or token consumption is reported, making it impossible to assess the practical cost of deploying this system."
     },
     {
       "flag": "Potential benchmark contamination not discussed",
       "detail": "The evaluation uses student articles from a 2024 NTU course published in an EMNLP 2024 paper (Chiang et al., 2024). GPT-4.1-mini and Gemini-2.5 models likely have training data cutoffs that include this published work, but no contamination analysis is provided."
     },
     {
       "flag": "Model version ambiguity",
       "detail": "The paper uses 'gpt-4.1-mini', 'GPT-5-mini', 'GPT-4.1-nano', 'Gemini-2.5-flash', etc. as model identifiers. These are not standard API versioning identifiers with snapshot dates, making exact reproduction difficult as these models may have changed since the experiments were run."
     },
     {
       "flag": "Training data is LLM-generated",
       "detail": "The training set consists of 50 GPT-generated benign articles, not authentic student submissions. This raises a question about whether the optimization might overfit to GPT-style text distributions, though the real-world evaluation on actual student submissions partially mitigates this concern."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language model as an assignment evaluator: Insights, feedback, and challenges in a 1000+ student course",
       "authors": [
         "Cheng-Han Chiang",
         "Wei-Chih Chen",
         "Chun-Yi Kuan",
         "Chienchou Yang",
         "Hung-Yi Lee"
       ],
       "year": 2024,
       "relevance": "Provides the real-world dataset and Human-Crafted Prompt baseline that AEGIS is evaluated against; directly relevant as the predecessor work on LLM-based grading and prompt injection defense."
     },
     {
       "title": "Automatic prompt optimization with 'gradient descent' and beam search",
       "authors": [
         "Reid Pryzant",
         "Dan Iter",
         "Jerry Li",
         "Yin Tat Lee",
         "Chenguang Zhu",
         "Michael Zeng"
       ],
       "year": 2023,
       "arxiv_id": "2305.03495",
       "relevance": "Introduces the TGO (Textual Gradient Optimization) framework that AEGIS extends, making it directly foundational to the paper's methodology."
     },
     {
       "title": "Detecting language model attacks with perplexity",
       "authors": [
         "Gabriel Alon",
         "Michael Kamfonas"
       ],
       "year": 2023,
       "arxiv_id": "2308.14132",
       "relevance": "Serves as one of three baselines evaluated against in Table 1; relevant to prompt injection defense benchmarking."
     },
     {
       "title": "Llama guard: LLM-based input-output safeguard for human-AI conversations",
       "authors": [
         "Hakan Inan",
         "Kartikeya Upasani",
         "Jianfeng Chi"
       ],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "Serves as one of three baselines (LLaMA 3.1 Guard) evaluated against in Table 1; relevant to LLM safety and content moderation."
     },
     {
       "title": "Defending against indirect prompt injection attacks with spotlighting",
       "authors": [
         "Keegan Hines",
         "Gary Lopez",
         "Matthew Hall",
         "Federico Zarfati",
         "Yonatan Zunger",
         "Emre Kiciman"
       ],
       "year": 2024,
       "arxiv_id": "2403.14720",
       "relevance": "Discussed as a representative training-free defense approach (input-level structure encoding) in the related work section."
     },
     {
       "title": "Robust prompt optimization for defending language models against jailbreaking attacks",
       "authors": [
         "Andy Zhou",
         "Bo Li",
         "Haohan Wang"
       ],
       "year": 2024,
       "relevance": "Presents RPO, the most closely related adversarial optimization framework for prompt defense, but operates in a white-box setting in contrast to AEGIS's black-box approach."
     },
     {
       "title": "Survival of the safest: Towards secure prompt optimization through interleaved multi-objective evolution",
       "authors": [
         "Ankita Sinha",
         "Wendi Cui",
         "Kamalika Das",
         "Jiaxin Zhang"
       ],
       "year": 2024,
       "arxiv_id": "2410.09652",
       "relevance": "Closely related multi-objective evolutionary prompt optimization framework that inspired AEGIS's multi-objective scoring approach."
     },
     {
       "title": "Automatic prompt optimization via heuristic search: A survey",
       "authors": [
         "Wendi Cui",
         "Jiaxin Zhang",
         "Zhuohang Li",
         "Hao Sun",
         "Damien Lopez",
         "Kamalika Das",
         "Bradley A Malin",
         "Sricharan Kumar"
       ],
       "year": 2025,
       "arxiv_id": "2502.18746",
       "relevance": "Survey of prompt optimization techniques that provides the taxonomy (heuristic-based, static vs. evolving environments) used to situate AEGIS in the literature."
     },
     {
       "title": "MELON: Indirect prompt injection defense via masked re-execution and tool comparison",
       "authors": [
         "Kaijie Zhu",
         "Xianjun Yang",
         "Jindong Wang",
         "Wenbo Guo",
         "William Yang Wang"
       ],
       "year": 2025,
       "relevance": "Discussed as a representative behavioral consistency checking defense approach in the related work; directly relevant to prompt injection defense comparisons."
     },
     {
       "title": "PromptArmor: Simple yet effective prompt injection defenses",
       "authors": [
         "Tianneng Shi",
         "Kaijie Zhu",
         "Zhun Wang"
       ],
       "year": 2025,
       "arxiv_id": "2507.15219",
       "relevance": "Discussed as a representative LLM-based detection prompt defense approach in the related work section."
     }
   ]
 }