ai-research-survey

scan.json (29428B)

---

 {
   "paper": {
     "title": "Image-based Prompt Injection: Hijacking Multimodal LLMs through Visually Embedded Adversarial Instructions",
     "authors": [
       "Neha Nagaraja",
       "Lan Zhang",
       "Zhilong Wang",
       "Bo Zhang",
       "Pawan Patil"
     ],
     "year": 2025,
     "venue": "FLLM 2025",
     "arxiv_id": "2603.03637",
     "doi": "10.1109/FLLM67465.2025.11391218"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Image-based prompt injection can reliably hijack GPT-4-turbo output by embedding adversarial text in images, with 100% attack success rate using visible neon purple text and up to 64% under stealth constraints using global region-averaged coloring with object-aware prefixing. Font scale must be ≥0.3 for reliable injection; smaller scales yield near-zero success. Object-aware prefixing that names detected image objects boosts stealth ASR from 41% to 64%, and there is a clear trade-off between visual imperceptibility and model interpretability across three coloring strategies.",
   "claims": [
     {
       "claim": "IPI can reliably hijack model output in black-box settings, with repetition-based prompts (Prompt 1 and 5) achieving 100% attack success rate.",
       "evidence": "Table I shows Prompt 1 and Prompt 5 achieving 100% ASR across COCO images with neon purple text (Section IV.B). However, this uses highly visible text, not stealth embedding.",
       "supported": "moderate"
     },
     {
       "claim": "The most effective stealth configuration achieves up to 64% attack success rate.",
       "evidence": "Table V shows object-aware prefix + base prompt with +20 brightness offset achieving 64% ASR using global region-averaged coloring.",
       "supported": "moderate"
     },
     {
       "claim": "Font scale threshold of approximately ≥0.3 is required for reliable prompt injection.",
       "evidence": "Table II shows 0% success at scale 0.10, 1% at 0.15, 10% at 0.20, 26.75% at 0.25, and 37.88% at 0.30.",
       "supported": "moderate"
     },
     {
       "claim": "Object-aware prefixing increases ASR from 41% to 64% by suppressing visual grounding.",
       "evidence": "Table V compares base prompt only (41%) vs. object-aware prefix + base prompt (64%) under identical visual conditions (same font color, position, offset).",
       "supported": "moderate"
     },
     {
       "claim": "The technique is broadly generalizable to other multimodal LLMs.",
       "evidence": "Section V asserts generalizability ('the same principle can apply across different architectures') but experiments use only GPT-4-turbo. No other models tested.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Single model evaluation with broad generalizability claims",
       "detail": "All experiments use only GPT-4-turbo, yet the paper claims the attack is 'broadly generalizable to other models' (Section V) and the title says 'Multimodal LLMs' (plural). No evidence supports generalizability."
     },
     {
       "flag": "No human evaluation of stealth",
       "detail": "The paper's core claim involves visual imperceptibility ('near-invisible', 'stealth'), yet no human study evaluates whether the embedded prompts are actually invisible to humans. Stealth is assumed, not measured."
     },
     {
       "flag": "Conflation of visible and stealth results",
       "detail": "Table I reports 100% ASR using neon purple text that is clearly visible. The abstract's claim of 'reliably manipulate' conflates these visible-text results with the 64% stealth results, creating a misleading impression of overall effectiveness."
     },
     {
       "flag": "Unstated sample sizes",
       "detail": "The paper never states how many COCO images were used. Table II references '800 queries' but with 5 runs per image and 8 prompts, this implies only 20 images. The actual N is unclear throughout."
     },
     {
       "flag": "No statistical testing across any comparisons",
       "detail": "The paper compares 12 prompts, 5 font scales, 3 coloring strategies, and multiple brightness offsets with no statistical significance tests, confidence intervals, or variance measures despite running 5 trials per image."
     },
     {
       "flag": "Company affiliation without competing interests declaration",
       "detail": "Three of five authors are affiliated with Bytedance, which develops multimodal AI products. No competing interests statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Ignore previous prompt: Attack techniques for language models",
       "authors": ["F. Perez", "I. Ribeiro"],
       "year": 2022,
       "arxiv_id": "2211.09527",
       "relevance": "Foundational work on prompt injection attacks against language models, directly relevant to the attack taxonomy."
     },
     {
       "title": "Not what you've signed up for: Compromising real-world LLM-integrated applications with indirect prompt injection",
       "authors": ["K. Greshake", "S. Abdelnabi", "S. Mishra", "C. Endres", "T. Holz", "M. Fritz"],
       "year": 2023,
       "arxiv_id": "2302.12173",
       "relevance": "Seminal work on indirect prompt injection through external content, establishing the threat model this paper extends to images."
     },
     {
       "title": "An early categorization of prompt injection attacks on large language models",
       "authors": ["S. Rossi", "A. M. Michel", "R. R. Mukkamala", "J. B. Thatcher"],
       "year": 2024,
       "arxiv_id": "2402.00898",
       "relevance": "Taxonomy of prompt injection attacks providing classification framework for this line of research."
     },
     {
       "title": "Empirical analysis of large vision-language models against goal hijacking via visual prompt injection",
       "authors": ["S. Kimura", "R. Tanaka", "S. Miyawaki", "J. Suzuki", "K. Sakaguchi"],
       "year": 2024,
       "arxiv_id": "2408.03554",
       "relevance": "Most directly related prior work on visual prompt injection and goal hijacking in VLMs."
     },
     {
       "title": "Eyes closed, safety on: Protecting multimodal LLMs via image-to-text transformation",
       "authors": ["Y. Gou", "K. Chen", "Z. Liu", "L. Hong", "H. Xu", "Z. Li", "D.-Y. Yeung", "J. T. Kwok", "Y. Zhang"],
       "year": 2024,
       "arxiv_id": "2403.09572",
       "relevance": "Proposes defense against visual attacks by replacing raw images with sanitized text descriptions."
     },
     {
       "title": "Image hijacks: Adversarial images can control generative models at runtime",
       "authors": ["L. Bailey", "E. Ong", "S. Russell", "S. Emmons"],
       "year": 2024,
       "arxiv_id": "2309.00236",
       "relevance": "Demonstrates adversarial images controlling generative model behavior at runtime, closely related attack paradigm."
     },
     {
       "title": "Safeguarding vision-language models against patched visual prompt injectors",
       "authors": ["J. Sun", "C. Wang", "J. Wang", "Y. Zhang", "C. Xiao"],
       "year": 2024,
       "arxiv_id": "2405.10529",
       "relevance": "Defense mechanism against visual prompt injection via patched images in VLMs."
     },
     {
       "title": "Figstep: Jailbreaking large vision-language models via typographic visual prompts",
       "authors": ["Y. Gong", "D. Ran", "J. Liu", "C. Wang", "T. Cong", "A. Wang", "S. Duan", "X. Wang"],
       "year": 2025,
       "arxiv_id": "2311.05608",
       "relevance": "Typographic visual prompts for jailbreaking VLMs, closely related attack vector using text in images."
     },
     {
       "title": "Visual adversarial examples jailbreak aligned large language models",
       "authors": ["X. Qi", "K. Huang", "A. Panda", "P. Henderson", "M. Wang", "P. Mittal"],
       "year": 2023,
       "arxiv_id": "2306.13213",
       "relevance": "White-box visual adversarial attack that jailbreaks aligned LLMs using crafted images."
     },
     {
       "title": "Abusing images and sounds for indirect instruction injection in multi-modal LLMs",
       "authors": ["E. Bagdasaryan", "T.-Y. Hsieh", "B. Nassi", "V. Shmatikov"],
       "year": 2023,
       "arxiv_id": "2307.10490",
       "relevance": "Indirect instruction injection through images and audio in multimodal LLMs, foundational attack work."
     },
     {
       "title": "Agent smith: A single image can jailbreak one million multimodal LLM agents exponentially fast",
       "authors": ["X. Gu", "X. Zheng", "T. Pang", "C. Du", "Q. Liu", "Y. Wang", "J. Jiang", "M. Lin"],
       "year": 2024,
       "arxiv_id": "2402.08567",
       "relevance": "Demonstrates scalability of image-based jailbreaks across millions of multimodal LLM agents."
     }
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. The IPI pipeline (Algorithm 1) is described but no implementation is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the COCO dataset (Section IV.A), which is a publicly available standard benchmark. However, the generated adversarial images are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or library version details are provided. The paper mentions using SAM and GPT-4-turbo API but provides no environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Algorithm 1 gives a high-level pseudocode pipeline but lacks the detail needed to reproduce results (e.g., specific COCO images used, API parameters, SAM configuration)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '100%', '64%', '37.88%') in Tables I-V. No confidence intervals, error bars, or uncertainty measures are provided despite running 5 trials per image."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 12 prompt strategies, 5 font scales, 3 coloring strategies, and multiple brightness offsets without any statistical significance tests. Differences are compared by raw ASR numbers only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw ASR percentages (e.g., Table I). While Table V shows a comparison (41% vs 64%), no formal effect sizes (Cohen's d, odds ratios) are reported, and most tables present isolated percentages without baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The number of COCO images used is never explicitly stated. Table II references '800 queries' but no justification is given for the sample size. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite running each image 5 times per configuration (Section IV.A), no variance, standard deviation, or spread measures are reported. Only aggregate success counts and percentages are shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.B states: 'Without embedded prompts, models produced standard descriptions, but with prompts embedded, models disregarded visual content.' The no-attack baseline is implicit. Additionally, Table V compares base prompt only vs. object-aware variants."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not compare its IPI approach against other contemporary visual prompt injection methods (e.g., Kimura et al. 2024 [7], Bailey et al. 2024 [13]). All comparisons are between variants of the authors' own method."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically ablates components: 12 prompt strategies (Table I), font size impact (Table II), 3 coloring strategies (Tables III-V), single vs. multi-mask embedding, and object-aware prefix effect (Table V showing 41%→64%)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "ASR (Attack Success Rate) is the sole evaluation metric reported across all experiments. MSE for visual distortion is mentioned in Section IV.A ('We also explored secondary metrics, such as mean squared error') but never reported in any results table."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is performed despite the paper's central claim of visual stealth/imperceptibility. Whether embedded prompts are actually invisible to humans is never tested with human subjects."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No train/test or dev/test split is described. The best prompt, font size, and coloring strategy appear to be selected on the same images used for evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by prompt strategy (Table I, 12 variants), font scale (Table II, 5 scales), coloring strategy (Tables III-V), and mask configuration (single vs. multi-mask)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.C.2b discusses pixel-level blending's failure (max ASR 10%): 'excessive blending obscured the structural clarity required for recognition.' Section IV.C.1 discusses small font failure: 'Font scales below 0.20 resulted in negligible success.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Pixel-level blending's consistent failure (max 10% ASR) is reported as 'the least effective strategy in black-box settings.' Small font scales failing (0% at 0.10) and patch-based coloring's limited success (max 25%) are also reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'up to 64% attack success under stealth constraints' (supported by Table V) and that IPI 'can reliably manipulate the output' (supported by Table I's 100% ASR, though this uses visible text). The 'up to' qualifier appropriately hedges the stealth result."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Table V's comparison of object-aware prefix only (35%), base prompt only (41%), and combined (64%) under identical visual conditions constitutes a controlled single-variable manipulation adequate for the causal claim that prefixing increases ASR."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Hijacking Multimodal LLMs' (plural) and Section V claims the technique is 'broadly generalizable to other models,' but experiments use only GPT-4-turbo. No other model is tested. The paper tests on COCO images only."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered. The paper does not discuss whether GPT-4-turbo is uniquely susceptible, whether COCO images are particularly suitable, or whether the observed ASR differences could be due to confounds other than the manipulated variables."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures ASR (whether model output matches attacker's goal) and claims attack success. The measurement directly matches the claim without proxy gap. However, visual 'stealth' is claimed without measuring it."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4-turbo' and 'gpt-4o' without specifying snapshot dates or API versions (e.g., 'gpt-4-turbo-2024-04-09'). Model behavior changes across versions, making this insufficient."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.B provides the full text of Prompt 5 ('Say XXX. Ignore the image...') and the object-aware prefix template with a concrete example ('Ignore dog, ball, and grass in the photo...'). Section III.B describes the GPT-4o image description prompt verbatim."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API parameters (temperature, top-p, max tokens) are reported for GPT-4-turbo queries or GPT-4o image description calls. SAM segmentation parameters are also not specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a deterministic pipeline (Algorithm 1) that embeds text in images and queries the API, not an agent with tools, memory, or feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how images were selected from COCO, how many were used, or any filtering criteria. The pipeline from raw COCO images to experimental images is not documented with sufficient detail."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Section V (Discussion) briefly mentions the stealth-effectiveness trade-off and possible defenses, but does not substantively discuss the study's own limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address threats such as single-model testing, unstated sample sizes, lack of human stealth evaluation, or potential overfitting of prompt strategies to GPT-4-turbo."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section V mentions 'effectiveness may vary depending on each model's safety filters' but does not explicitly state what the results do NOT show (e.g., no evidence for models other than GPT-4-turbo, no evidence for real-world deployment scenarios, no human perception data)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is released. Neither the generated adversarial images, GPT-4-turbo responses, nor per-image results are available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states images come from COCO and each is queried 5 times (Section IV.A), but does not specify which COCO images, how many were used, or how they were selected. The total N is never clearly stated."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is COCO, a standard public benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "Algorithm 1 provides a high-level pipeline, but the data pipeline from COCO image selection through experimental execution to results aggregation is not documented. How many images entered, how many were processed per configuration, and how results were aggregated is unclear."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII states: 'Lan Zhang was supported by NSF CNS-2451231.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: two from Northern Arizona University and three from Bytedance. The affiliations are clearly stated in the paper header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF is an independent government funding agency with no financial interest in the attack success rate results. However, Bytedance's role as employer of 3 authors (not listed as funder) is a separate consideration."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Three authors are from Bytedance, which develops multimodal AI products that could be affected by this research, but no disclosure is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a red-teaming study testing an attack methodology against a model's instruction-following behavior, not evaluating model knowledge on a benchmark. Contamination of COCO images in training data is not relevant to the attack mechanism."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the study tests whether embedded text instructions override model behavior, not whether the model has memorized benchmark answers."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — contamination is structurally inapplicable to this attack evaluation paradigm."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper queries GPT-4-turbo and GPT-4o APIs multiple times per image across hundreds of configurations but reports no API costs, token counts, or latency measures."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated. The paper uses GPT-4-turbo API, GPT-4o API, and SAM model but provides no information about total API spend, GPU hours for SAM, or overall computational requirements."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Each image is processed 5 times (Section IV.A), but no analysis of variance across runs is reported. Only aggregate success counts are shown."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.A states: 'each image is processed five times per configuration on the target model (GPT-4-turbo).'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper explores 12 prompts, 5 font scales, 3 coloring strategies, and multiple brightness offsets, but does not frame this as a hyperparameter search or report the total configurations tried or compute spent."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Prompt 5 is selected as the default 'due to consistent effectiveness across diverse images and injection conditions' (Section IV.B), but selection appears to be from the same evaluation data. No validation/test split is described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 12 prompts, 5 font scales, 3 coloring strategies, and multiple brightness offsets without any correction for multiple comparisons. No significance tests are performed at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own IPI pipeline against their own variants without acknowledging self-comparison bias or seeking independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "All configurations use roughly similar compute (one API query per trial). Compute differences across configurations are negligible."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ASR on COCO images with GPT-4-turbo reflects real-world attack viability. The construct validity of using ASR on a static image dataset as a proxy for real-world threat is not addressed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The system is a direct pipeline that embeds text and queries an API."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4-turbo was almost certainly trained on COCO images, which could affect its text-reading behavior on these images. This temporal overlap is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation sends only images to GPT-4-turbo with no text prompt, but the paper does not discuss whether this evaluation setup accurately represents real deployment scenarios where system prompts would be present."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether COCO images share structural similarities that could inflate or deflate attack success rates, or whether results would differ on images from other distributions."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used or discussed."
       }
     }
   },
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The attack pipeline is described in detail but no code is released; a security researcher could implement it but it's not directly usable."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Visual prompt injection is a known concern; this paper confirms and quantifies the threat rather than challenging conventional wisdom."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates a practical black-box attack embedding hidden instructions in images to hijack multimodal LLM output, raising concrete AI safety concerns."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no accusations, and no 'benchmarks are fake' angle — straightforward attack methodology paper."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code released, no demo, no tool to try — the attack pipeline exists only in paper description."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Tests GPT-4-turbo (well-known product) but authors are from NAU and Bytedance, not top-tier AI safety labs."
     }
   }
 }