ai-research-survey

scan.json (22436B)

---

 {
   "paper": {
     "title": "Attacking LLMs and AI Agents: Advertisement Embedding Attacks Against Large Language Models",
     "authors": ["Qiming Guo", "Jinwen Tang", "Xingran Huang"],
     "year": 2025,
     "venue": "IEEE",
     "arxiv_id": "2508.17674",
     "doi": "10.48550/arXiv.2508.17674"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or supplementary materials are mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The attack data (malicious Q&A pairs) and fine-tuning datasets used are not released or made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The only hardware mention is 'a local RTX 4070 graphics card' for LoRA fine-tuning. No software versions, library versions, requirements.txt, or environment specifications are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The attack prompts are given inline but there are no instructions for setting up the attack pipeline or reproducing the fine-tuning experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No quantitative results with confidence intervals or error bars are reported. The paper makes qualitative claims about attack effectiveness ('almost 100%' success) without formal measurement."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. The paper claims attacks are effective based on qualitative demonstrations without any formal comparison or testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. The claim of 'almost 100%' reproduction of preset responses is the closest to a quantitative result, but no formal measurement methodology, sample size, or baseline context is provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The number of test queries is not stated, let alone justified. The paper shows a handful of example Q&A pairs in Figure 2 but does not describe a systematic evaluation with a defined sample size."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across any experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included. The paper does not compare AEA to other attack methods or compare the defense to other defense mechanisms."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are compared at all, so contemporariness is moot."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The attack has multiple components (prompt design, data content, model choice) but no ablation study examines which components contribute to attack success."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "No formal metrics are defined or reported. The paper relies on qualitative demonstrations (showing example outputs) rather than measurable metrics like attack success rate, stealthiness, or detection rate."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. Claims about stealthiness and user deception are made without any user study to verify whether humans can detect the embedded content."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No formal test set is used. The paper demonstrates attacks on a handful of example queries shown in Figure 2 without a systematic held-out evaluation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "No per-category breakdown is provided. The paper does not analyze attack success by question type, content category, or model."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases are discussed. The paper does not describe scenarios where the attacks fail or where the defense is insufficient."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. Every experiment shown is a success case. The defense is acknowledged to not work against parameter-level attacks, but this is a design limitation rather than a negative experimental result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims AEA operate through 'two low-cost vectors' and presents 'an initial prompt-based self-inspection defense that mitigates these injections.' The low-cost claim is supported only by a single anecdote (1 hour on RTX 4070). The defense claim is supported only by describing a prompt without systematic evaluation of its effectiveness."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims that AEA 'cause models to return covert ads, propaganda, or hate speech' and that the defense 'mitigates these injections.' These are demonstrated with a few examples rather than a controlled experimental design that would support causal inference about attack mechanisms or defense effectiveness."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests on Gemini 2.5 for the SDP attack and LLaMA 3, Mistral 7B, Phi-3 for fine-tuning, but makes broad claims about LLM security generally. The title 'Attacking LLMs and AI Agents' and claims about the threat being as 'prevalent as web viruses' far exceed the narrow demonstrations provided."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The paper does not consider whether the observed attack success might be specific to certain model architectures, prompt structures, or whether existing safety mechanisms might already mitigate these attacks in practice."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Gemini 2.5', 'LLaMA 3', 'Mistral 7B', and 'Phi-3' but does not provide specific version identifiers, snapshot dates, or API versions for any of them. 'Gemini 2.5' is a marketing name without a precise version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The attack prompt is provided verbatim in Section IV-B1 and the defense prompt is provided in Section IV-C. These are the actual prompts used, not templates or natural language descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for either the LLM API calls (temperature, top-p) or the LoRA fine-tuning (learning rate, rank, alpha, epochs). Only the GPU model (RTX 4070) and training time (1 hour) are mentioned."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. The attack is a prompt injection / fine-tuning attack, not an agentic workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The attack data (malicious Q&A pairs) and fine-tuning data are not described in detail. The paper mentions 'simulated attacker data included falsified history and hate speech' without documenting how this data was prepared, its format, or its size."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion acknowledges 'more research is needed' but does not discuss specific limitations of the study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address internal or external validity concerns."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what it does NOT show or which settings/models/scenarios are excluded from its claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (attack prompts corpus, fine-tuning datasets, model outputs) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions observing 'threat actors in underground markets distributing compromised models designed to increase traffic to specific gambling websites' but provides no details about how these observations were made, when, or what evidence was collected."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The experiments involve running prompts against LLM APIs and fine-tuning models."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline is documented. The process from creating attack data to executing attacks to measuring results is not described systematically."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The acknowledgment section only mentions 'ChatGPT refined the writing for grammatical accuracy' with no funding information."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Texas A&M University-Corpus Christi, University of Missouri, and University of California-Riverside."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of the funder cannot be assessed. The absence of a funding disclosure statement is a gap."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests attacks (prompt injection and model poisoning) rather than evaluating a pre-trained model's knowledge capability on a benchmark. Contamination in the traditional sense is not applicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model on a benchmark. It tests attack vectors, so train/test overlap is not a relevant concern."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark is used for capability evaluation. The paper demonstrates attacks using custom adversarial inputs, not standardized benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims attacks are 'low-cost' and mentions using an RTX 4070 for 1 hour for LoRA fine-tuning, but does not report API costs for the SDP attack path, token counts, or any systematic cost analysis."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'a local RTX 4070 graphics card and took 1 hour' for fine-tuning but does not provide a total computational budget, GPU hours across all experiments, or API costs."
       }
     }
   },
   "claims": [
     {
       "claim": "AEA can stealthily inject promotional or malicious content into LLM outputs through service distribution platform hijacking.",
       "evidence": "Section IV-B1 demonstrates the attack on Google Gemini 2.5 with a specific attack prompt, showing modified outputs in Figure 2.",
       "supported": "weak"
     },
     {
       "claim": "AEA via model distribution platforms achieve 'almost 100%' reproduction of preset attack responses through LoRA fine-tuning.",
       "evidence": "Section IV-B2 states 'Parameter-tuning attacks proved highly effective, reproducing almost 100% of the preset responses from the attacker dataset' on LLaMA 3, Mistral 7B, and Phi-3, but provides no formal measurement methodology, sample size, or detailed results.",
       "supported": "weak"
     },
     {
       "claim": "The LoRA fine-tuning attack can be implemented with only a local RTX 4070 GPU in 1 hour.",
       "evidence": "Section IV-B2 states this directly, supporting the low-cost claim for this specific attack vector.",
       "supported": "moderate"
     },
     {
       "claim": "A prompt-based self-inspection defense can effectively defend against API-based AEA attacks.",
       "evidence": "Section IV-C describes the defense prompt and claims it 'can effectively defend against attacks based on API Providers' but provides no quantitative evaluation, success rate, or systematic testing of the defense.",
       "supported": "unsupported"
     },
     {
       "claim": "Current service providers (including Google Gemini) remain inadequately prepared to defend against AEA.",
       "evidence": "Section I and IV-B1 demonstrate that Gemini 2.5 was 'successfully manipulated by our attack prompts' but this is shown through a single demonstration, not a systematic evaluation across providers.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["case-study"],
   "key_findings": "The paper introduces Advertisement Embedding Attacks (AEA), a class of attacks that inject promotional or malicious content into LLM outputs via two vectors: hijacking service distribution platforms to prepend adversarial prompts, and publishing backdoored open-source model checkpoints fine-tuned with attacker data. The authors demonstrate attacks on Gemini 2.5 and three open-source models (LLaMA 3, Mistral 7B, Phi-3), and propose a basic prompt-based defense. However, the evaluation consists entirely of qualitative demonstrations with no systematic measurement of attack success rates, stealthiness, or defense effectiveness.",
   "red_flags": [
     {
       "flag": "No quantitative evaluation",
       "detail": "The paper provides only qualitative demonstrations (a few example outputs in Figure 2) with no formal metrics, success rates, or systematic evaluation. Claims of 'almost 100%' attack success are unsupported by rigorous measurement."
     },
     {
       "flag": "Unsupported defense effectiveness claim",
       "detail": "The defense method is described with a single example prompt and claimed to 'effectively defend' without any evaluation. No success rate, false positive rate, or systematic testing is provided."
     },
     {
       "flag": "Overclaimed scope",
       "detail": "The paper tests on Gemini 2.5 for one attack vector and three models for another, but makes sweeping claims about LLM security broadly and predicts AEA 'will become as prevalent as web viruses' without evidence for such generalization."
     },
     {
       "flag": "No baselines or comparisons",
       "detail": "The attacks are not compared to existing prompt injection or backdoor attack methods. The defense is not compared to any existing defense mechanisms."
     },
     {
       "flag": "Missing limitations section",
       "detail": "The paper has no limitations, threats-to-validity, or ethical considerations section, despite proposing and demonstrating attacks that could be misused."
     },
     {
       "flag": "Novelty claim questionable",
       "detail": "The paper claims to define a 'newly discovered' attack class, but prompt injection and model poisoning/backdoor attacks are well-established in the literature. The paper does not clearly differentiate AEA from these existing attack categories beyond the specific motivation of advertising."
     }
   ],
   "cited_papers": [
     {
       "title": "Language models are few-shot learners",
       "authors": ["T. Brown", "B. Mann", "N. Ryder"],
       "year": 2020,
       "relevance": "Foundational GPT-3 paper relevant to understanding LLM capabilities that enable the attacks discussed."
     },
     {
       "title": "On the opportunities and risks of foundation models",
       "authors": ["R. Bommasani", "D. A. Hudson", "E. Adeli"],
       "year": 2021,
       "arxiv_id": "2108.07258",
       "relevance": "Comprehensive survey of foundation model risks and opportunities relevant to LLM security assessment."
     },
     {
       "title": "Sleeper agents: Training deceptive LLMs that persist through safety training",
       "authors": ["E. Hubinger", "C. Denison", "J. Mu"],
       "year": 2024,
       "arxiv_id": "2401.05566",
       "relevance": "Directly relevant to backdoor and deceptive behavior in LLMs, closely related to the model poisoning attack vector in AEA."
     },
     {
       "title": "Jailbroken: How does LLM safety training fail?",
       "authors": ["A. Wei", "N. Haghtalab", "J. Steinhardt"],
       "year": 2023,
       "relevance": "Foundational work on LLM jailbreaking attacks relevant to understanding prompt-based attack mechanisms."
     },
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": ["K. Greshake", "S. Abdelnabi", "S. Mishra"],
       "year": 2023,
       "relevance": "Directly relevant indirect prompt injection attack paper that establishes the broader threat category AEA falls under."
     },
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["A. Zou", "Z. Wang", "N. Carlini"],
       "year": 2023,
       "arxiv_id": "2307.15043",
       "relevance": "Key paper on adversarial attacks against LLMs, directly relevant to understanding attack transferability."
     },
     {
       "title": "Jailbreaking ChatGPT via prompt engineering: An empirical study",
       "authors": ["Y. Liu", "G. Deng", "Z. Xu"],
       "year": 2023,
       "arxiv_id": "2305.13860",
       "relevance": "Empirical study of prompt-based attacks on ChatGPT, directly relevant to the prompt injection attack vector."
     },
     {
       "title": "Weight poisoning attacks on pre-trained models",
       "authors": ["K. Kurita", "P. Michel", "G. Neubig"],
       "year": 2020,
       "arxiv_id": "2004.06660",
       "relevance": "Foundational work on weight poisoning in pre-trained models, directly relevant to the model distribution platform attack vector."
     },
     {
       "title": "PoisonPrompt: Backdoor attack on prompt-based large language models",
       "authors": ["H. Yao", "J. Lou", "Z. Qin"],
       "year": 2024,
       "relevance": "Directly relevant backdoor attack on prompt-based LLMs, closely related to the attack methodology in AEA."
     },
     {
       "title": "Scalable extraction of training data from (production) language models",
       "authors": ["M. Nasr", "N. Carlini", "J. Hayase"],
       "year": 2023,
       "arxiv_id": "2311.17035",
       "relevance": "Important work on LLM data extraction attacks, relevant to the broader threat landscape of LLM security."
     },
     {
       "title": "A survey on evaluation of large language models",
       "authors": ["Y. Chang", "X. Wang", "J. Wang"],
       "year": 2024,
       "relevance": "Survey on LLM evaluation methodology relevant to assessing how LLM security evaluations should be conducted."
     }
   ]
 }