ai-research-survey

scan.json (32837B)

---

 {
   "paper": {
     "title": "Fun-tuning: Characterizing the Vulnerability of Proprietary LLMs to Optimization-based Prompt Injection Attacks via the Fine-Tuning Interface",
     "authors": [
       "Andrey Labunets",
       "Nishit V. Pandya",
       "Ashish Hooda",
       "Xiaohan Fu",
       "Earlence Fernandes"
     ],
     "year": 2025,
     "venue": "IEEE Symposium on Security and Privacy",
     "arxiv_id": "2501.09798",
     "doi": "10.1109/SP61157.2025.00121"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The fine-tuning interface of proprietary LLMs leaks loss-like information that can guide optimization-based prompt injection attacks on closed-weight models. Using the Gemini fine-tuning API with a very small learning rate, the authors achieved 65% and 82% attack success rates on Gemini 1.5 Flash and 1.0 Pro respectively on the PurpleLlama benchmark. The attacks cost under $10 total and transfer across Gemini model variants with 50-90% success rates. Google deployed mitigations (minimum learning rate cap, minimum batch size of 4) following responsible disclosure.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code is released at https://github.com/earlence-security/fun-tuning, stated in the Disclosure and Ethics section: 'Code is available at https://github.com/earlence-security/fun-tuning.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses the PurpleLlama CyberSecEval benchmark (publicly available). PPL40 is a 40-example subset sampled from the public benchmark with the sampling criteria described in Section 6.1."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification (requirements.txt, Dockerfile, or detailed dependency listing) is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides algorithmic pseudocode (Algorithms 1 and 2) and attack configuration details, but no step-by-step reproduction instructions (README with commands, reproduction scripts) are described in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported for all ASR results: e.g., '82.0 ± 4.2' (Table 3), '65.3 ± 3.8' (Table 4), and transfer evaluation results in Tables 5-6 all include ± values."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No formal significance tests (p-values, t-tests, etc.) are used. The paper claims improvements are 'outside of standard deviation' (Tables 3-4 captions) but this is not a formal statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Improvement factors over baseline are reported: 1.9x for Gemini 1.0 Pro and 2.4x for Gemini 1.5 Flash (Tables 3-4). Absolute ASR values with baselines provide context for the magnitude of improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 40 examples (PPL40) from the 56-example PurpleLlama benchmark is not justified with power analysis or reasoning about statistical adequacy. The paper states they 'randomly sampled 40 indirect prompt injection examples' for 'quicker exploration' (Section 6.1)."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Variance is reported across 20 scoring runs for primary attacks and 5 runs for transfer evaluation: e.g., '82.0 ± 4.2' (Table 3). Section 6.2 states: 'we repeat this procedure 20 times... and report the mean and the standard deviation.'"
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines are included: (1) unmodified PurpleLlama injections without optimization (baseline ASR), and (2) an ablation attack using random numbers instead of training losses (Section 6.4, Tables 3-4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The PurpleLlama benchmark (2024) provides the base prompt injections, and the ablation baseline is a matched variant of the proposed method. The paper also contextualizes results against Google's own internal red-teaming evaluation (Section 6.5, reference to Gemini 1.5 report)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 describes a principled ablation: 'we only removed the effects of the fine-tuning procedure. Instead of receiving the true training losses for each candidate, this algorithm received random numbers. All other attack parameters... were kept the same.'"
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: attack success rate (ASR), improvement factor over baseline, number of fine-tuning requests, wall-clock time, and financial cost (Tables 3-4). Per-scenario and per-iteration breakdowns are also provided (Figs. 5-9)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated using GPT-4o as judge (Section 6.2). No human evaluation of attack outputs is performed, though the authors manually revised judge questions to be stricter."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The 40 PPL40 examples are used both for attack optimization and for measuring success rates. There is no separate held-out set. The transfer evaluation (Tables 5-6) tests on other models but the same 40 examples."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figures 8 and 9 show per-scenario ASR breakdowns (code, exercise, population, transaction, password, zubrowka, resume, employee) for both target models. Table 9 shows the attack category distribution."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 discusses failure cases: 'attacks are least successful for the password category' (~10-20% ASR), and code analysis fails against Gemini 1.5 Flash (40% ASR). They hypothesize reasons: 'Gemini models were trained to resist phishing in some way' and 'the newer model is significantly better at code analysis.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: password phishing largely fails, code analysis attacks fail on Gemini 1.5 Flash, and the ablation (random substitution) is 'surprisingly' effective at 43.8%/61.3% ASR (Section 6.5). They also note the Gemini 1.5 Flash attack is much slower (60 hours vs 15 hours per example)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '65% and 82%' ASR are supported by Tables 3 (82.0 ± 4.2%) and 4 (65.3 ± 3.8%). The claim about 'loss-like information' being useful for optimization is supported by Sections 4.2-4.3 (Figs. 2-4)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The central causal claim — that training loss signal improves attack success — is justified via the ablation study (Section 6.4) that removes only the loss signal while keeping all other components identical. Fun-tuning (82%) vs. ablation (61.3%) on Gemini 1.0 Pro isolates the contribution of the loss signal."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Proprietary LLMs' broadly, but only Google's Gemini models are tested. The meta-review explicitly flags: 'The attack has been demonstrated on a single service, so it is not yet known which other services might be vulnerable.' Table 7 discusses other APIs speculatively but without experimental evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses that random token substitution is 'surprisingly' effective (43.8%/61.3% ablation ASR), that some PurpleLlama attacks are naturally effective (27.5%/42.5% baseline), and that Gemini's safety tuning explains failures in specific categories (password, code). Section 6.5 considers multiple factors contributing to results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 explicitly addresses the gap between automated scoring and true attack success. They redesigned the PurpleLlama judge questions to be stricter, noting that original questions 'are overly permissive and result in too many false positives.' Table 2 shows a concrete false-positive/false-negative analysis."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model version identifiers are used throughout: 'gemini-1.5-flash-001-tuning', 'gemini-1.0-pro-001', 'gemini-1.5-pro-001', 'gemini-2.0-flash' (Section 6.3). For scoring, 'GPT-4o' is specified as the judge."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Example prompts are shown in Figures 1 and 10 with adversarial payloads highlighted. The prompts derive from the public PurpleLlama benchmark. Section 6.3 describes the chat format and token delimiters (<start_of_turn>, <end_of_turn>). Code is released containing full prompt implementations."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed hyperparameters in Section 6.3: 45 iterations, 2 restarts at iterations 15 and 30, 20-token prefix/suffix initialized with '!', 1000 candidates per iteration, batch size 1, learning rate ~10^-45, and inference temperature settings (default and 0)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a direct optimization algorithm that interacts with the fine-tuning API, not an agent pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 describes sampling 40 indirect injection examples from PurpleLlama's 56, excluding token smuggling category and non-standard encodings. Table 9 shows the distribution of attack types in PPL40 vs. PurpleLlama. Section 6.2 describes the judge question modifications."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. Section 7 (Discussion) discusses mitigations and attack universality across other APIs, but focuses on the mitigation landscape rather than methodological limitations of the study itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address issues like the representativeness of PPL40, the validity of GPT-4o as judge, potential confounds from Gemini safety training, or the approximate permutation recovery error affecting results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Section 7 states 'we established the feasibility of our method specifically against the Gemini API,' the paper does not systematically state what the results do NOT show. The meta-review's noteworthy concern — that only a single service was tested — is not explicitly addressed as a scope boundary in the paper."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw training loss data from fine-tuning experiments, intermediate optimization results, and per-example attack outputs are not released. Only aggregated ASR results are reported in the paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection is well described: PurpleLlama benchmark selection (Section 6.1), fine-tuning API interaction procedure (Sections 4-5), scoring methodology via GPT-4o (Section 6.2), and the 20-repetition scoring protocol."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is the standard PurpleLlama benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from PurpleLlama benchmark → PPL40 subset selection (with exclusion criteria stated) → attack optimization (Algorithms 1-2) → scoring via GPT-4o judge → ASR aggregation is documented across Sections 5-6."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section states: 'This work is supported in part by gifts from Amazon and Google and by NSF award 2312119.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: UC San Diego and University of Wisconsin Madison. No authors are affiliated with Google (the vendor whose product is attacked)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Google is both a funder ('gifts from... Google') and the vendor whose product (Gemini) is shown to be vulnerable. Although the findings go against Google's interest in Gemini being perceived as secure, Google has a clear financial stake in the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declarations appear in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff dates for the Gemini models used are not stated. This is relevant because PurpleLlama prompt patterns could appear in Gemini's safety training data, affecting measured ASR."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Gemini may have been specifically trained to resist PurpleLlama attacks. The paper notes 'Recent LLM products have been fine-tuned specifically to resist prompt injection attempts' (Section 3) but does not discuss benchmark-specific overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "PurpleLlama was publicly released in 2024 and Gemini models could have incorporated these patterns into safety training. This is not discussed, though it would deflate rather than inflate the reported ASR."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The Disclosure and Ethics section addresses responsible disclosure to Google, not human subjects research."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Per-example costs are reported: $0.18 for Gemini 1.0 Pro, $0.02 for Gemini 1.5 Flash (Tables 3-4). Total attack cost is stated: 'all attacks combined cost < $10.' Time per example: 15 hours (Gemini 1.0 Pro) and 60 hours (Gemini 1.5 Flash)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Compute budget is reported: 90 fine-tuning calls per example, each taking ~10 minutes (Gemini 1.0 Pro) or ~40 minutes (Gemini 1.5 Flash). Total time per attack: 15 hours and 60 hours respectively. Fine-tuning calls are free; inference costs are the financial budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The optimization algorithm uses random token sampling but is not run across multiple seeds. Variance is reported across 20 scoring runs (non-deterministic inference), but the optimization itself runs once per example with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 states: 'we repeat this procedure 20 times to evaluate the primary attack (5 times for the transfer evaluation) and report the mean and the standard deviation.' Optimization runs 45 iterations with 2 restarts."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No systematic hyperparameter search is described. Section 6.6 studies the effect of candidate set size locally but the main attack parameters (45 iterations, 2 restarts, 20-token prefix/suffix) are not justified through a search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The per-example best perturbation is selected as the maximum-scoring iteration across all 45 iterations (Section 6.3: 'we stored the perturbations found at each iteration so we could identify the best one'). Section 6.6 justifies the candidate set size of 1000 through local simulation (Fig. 7)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 10 model configurations, 3 methods, and 8 scenarios (Tables 3-6, Figs. 8-9) without any correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and evaluate their own attack against their own baselines and ablation. No acknowledgment of author-evaluation bias or use of independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figures 5-6 show ASR as a function of iterations (compute). Figure 7 shows attack success rate and loss as a function of candidate set size. The time and cost per example are reported alongside ASR."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses PurpleLlama as the benchmark without discussing whether it accurately represents real-world prompt injection risk. While Section 6.2 improves the judge questions, no discussion of whether the benchmark's scenarios, attack categories, or success criteria are valid measures of actual vulnerability."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The attack directly targets the model through the fine-tuning interface, not through any scaffold or agent framework."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. PurpleLlama was released in 2024 and the Gemini models may have incorporated these attack patterns into safety training before the experiments."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The fine-tuning interface setup could theoretically provide different information than what a real attacker would have in some deployment scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The 40 PPL40 examples may share structural similarities (from the same benchmark creators) that could inflate or deflate aggregate ASR."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used."
       }
     }
   },
   "claims": [
     {
       "claim": "Fun-tuning achieves 82% attack success rate on Gemini 1.0 Pro and 65% on Gemini 1.5 Flash on the PurpleLlama benchmark.",
       "evidence": "Tables 3 and 4 report 82.0 ± 4.2% ASR for Gemini 1.0 Pro and 65.3 ± 3.8% for Gemini 1.5 Flash, averaged over 20 scoring runs with standard deviations.",
       "supported": "strong"
     },
     {
       "claim": "The fine-tuning training loss is linearly proportional to average logprobs for long output sequences.",
       "evidence": "Section 4.2, Figure 3 shows R² approaching 1 as output length increases. The hypothesized relationship TrainingLoss(Y|X) = K(X) + l·AvgLogprobs(Y|X) is validated empirically with 10 open-ended questions.",
       "supported": "strong"
     },
     {
       "claim": "The training loss serves as a useful proxy signal for discrete optimization of adversarial prompts, even for short target strings.",
       "evidence": "Section 4.3, Figure 4 shows that the candidate selected by training loss is ranked among the top few by true logprobs, with distributions skewed left across three test questions with M=100 samples and N=10 candidates.",
       "supported": "moderate"
     },
     {
       "claim": "The improvement over baseline is due to the training loss signal, not other attack components.",
       "evidence": "Ablation study (Section 6.4) replaces training losses with random numbers while keeping all other parameters identical. Fun-tuning ASR (82.0%/65.3%) exceeds ablation ASR (61.3%/43.8%) with improvements outside standard deviation (Tables 3-4).",
       "supported": "strong"
     },
     {
       "claim": "Attacks computed against one Gemini model transfer to other Gemini models with relatively high success rates.",
       "evidence": "Tables 5-6 show transfer ASR: Gemini 1.0 Pro attacks transfer to other 1.0 Pro variants at >80% ASR and to 1.5 Flash at 49-56% ASR. Gemini 1.5 Flash attacks transfer to 2.0 Flash at ~90% ASR and to 1.0 Pro variants at >71% ASR.",
       "supported": "strong"
     },
     {
       "claim": "The approximate permutation recovery method misidentifies only ~7-8% of positions.",
       "evidence": "Table 8 (Appendix) shows normalized Hamming distance of 0.036-0.074 and Kendall correlation of 0.898-0.947 across training set sizes 100-1000, averaged over 5 approximate permutations.",
       "supported": "strong"
     },
     {
       "claim": "The Gemini fine-tuning API applies a fixed, deterministic permutation to training losses for a given dataset size.",
       "evidence": "Section 4.1 establishes this through controlled experiments with duplicated training data of known cardinalities and repeated fine-tuning requests with different data but same sizes.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Single vendor tested despite broad title",
       "detail": "The title says 'Proprietary LLMs' but only Google's Gemini is experimentally evaluated. Table 7 discusses other APIs (OpenAI, Anthropic) speculatively but provides no experimental evidence. The meta-review explicitly flags this as a noteworthy concern."
     },
     {
       "flag": "LLM-as-judge without validation",
       "detail": "GPT-4o is used as the sole judge for attack success. While the authors improved the judge questions (Section 6.2), they did not validate GPT-4o's judging accuracy against human evaluators, introducing potential systematic bias in ASR measurements."
     },
     {
       "flag": "No formal significance testing",
       "detail": "Comparisons between fun-tuning and baselines/ablation are described as having improvements 'outside of standard deviation' but no formal significance tests are applied. With only 40 examples and multiple comparisons across models and scenarios, this risks overstating the certainty of improvements."
     },
     {
       "flag": "Funder conflict",
       "detail": "Google is acknowledged as a funder ('gifts from... Google') while the paper demonstrates vulnerabilities in Google's Gemini product. Although the findings go against Google's interest, the funding relationship is not discussed as a potential conflict."
     },
     {
       "flag": "Ablation surprisingly strong",
       "detail": "The ablation (random substitution) achieves 43.8%/61.3% ASR vs. baseline 27.5%/42.5%, meaning much of the attack improvement comes from random token wrapping rather than loss-guided optimization. The incremental contribution of the loss signal (ablation→fun-tuning) is smaller than the random wrapping effect (baseline→ablation)."
     }
   ],
   "cited_papers": [
     {
       "title": "Universal and transferable adversarial attacks on aligned language models",
       "authors": ["A. Zou", "Z. Wang", "J. Z. Kolter", "M. Fredrikson"],
       "year": 2023,
       "relevance": "Foundational whitebox GCG algorithm for adversarial prompt optimization that this paper extends to the graybox setting."
     },
     {
       "title": "Cyberseceval 2: A wide-ranging cybersecurity evaluation suite for large language models",
       "authors": ["M. Bhatt", "S. Chennabasappa", "Y. Li"],
       "year": 2024,
       "arxiv_id": "2404.13161",
       "relevance": "PurpleLlama benchmark used as the primary evaluation dataset for prompt injection attacks in this paper."
     },
     {
       "title": "Not what you've signed up for: Compromising real-world llm-integrated applications with indirect prompt injection",
       "authors": ["K. Greshake", "S. Abdelnabi", "S. Mishra", "C. Endres", "T. Holz", "M. Fritz"],
       "year": 2023,
       "relevance": "Seminal work on indirect prompt injection attacks against LLM-integrated applications, establishing the threat model this paper builds upon."
     },
     {
       "title": "Neural exec: Learning (and learning from) execution triggers for prompt injection attacks",
       "authors": ["D. Pasquini", "M. Strohmeier", "C. Troncoso"],
       "year": 2024,
       "arxiv_id": "2403.03792",
       "relevance": "Whitebox prompt injection attack using GCG algorithm; fun-tuning adopts its prefix-suffix wrapping style for adversarial prompt structure."
     },
     {
       "title": "Query-based adversarial prompt generation",
       "authors": ["J. Hayase", "E. Borevkovic", "N. Carlini", "F. Tramèr", "M. Nasr"],
       "year": 2024,
       "arxiv_id": "2402.12329",
       "relevance": "Graybox attack using logprobs from inference endpoints; fun-tuning extends this by using fine-tuning loss when logprobs are unavailable."
     },
     {
       "title": "Covert malicious finetuning: Challenges in safeguarding llm adaptation",
       "authors": ["D. Halawi", "A. Wei", "E. Wallace", "T. T. Wang", "N. Haghtalab", "J. Steinhardt"],
       "year": 2024,
       "arxiv_id": "2406.20053",
       "relevance": "Demonstrates attacks on LLMs through fine-tuning APIs using encoded malicious data; orthogonal to fun-tuning which uses loss signals rather than weight updates."
     },
     {
       "title": "The instruction hierarchy: Training llms to prioritize privileged instructions",
       "authors": ["E. Wallace", "K. Xiao", "R. Leike", "L. Weng", "J. Heidecke", "A. Beutel"],
       "year": 2024,
       "arxiv_id": "2404.13208",
       "relevance": "Defense mechanism training LLMs to resist prompt injection by prioritizing system instructions over user/third-party inputs."
     },
     {
       "title": "Injecagent: Benchmarking indirect prompt injections in tool-integrated large language model agents",
       "authors": ["Q. Zhan", "Z. Liang", "Z. Ying", "D. Kang"],
       "year": 2024,
       "arxiv_id": "2403.02691",
       "relevance": "Benchmark for evaluating indirect prompt injection in tool-integrated LLM agents, directly relevant to the agentic attack scenarios discussed."
     },
     {
       "title": "Stealing part of a production language model",
       "authors": ["N. Carlini", "D. Paleka", "K. D. Dvijotham"],
       "year": 2024,
       "arxiv_id": "2403.06634",
       "relevance": "Demonstrates extraction of information from closed-weight LLMs through API access; related attack surface of reverse-engineering proprietary models."
     },
     {
       "title": "Jailbreaking black box large language models in twenty queries",
       "authors": ["P. Chao", "A. Robey", "E. Dobriban", "H. Hassani", "G. J. Pappas", "E. Wong"],
       "year": 2024,
       "arxiv_id": "2310.08419",
       "relevance": "PAIR algorithm for blackbox jailbreaking; represents the blackbox attack approach that fun-tuning's graybox method aims to outperform."
     },
     {
       "title": "Struq: Defending against prompt injection with structured queries",
       "authors": ["S. Chen", "J. Piet", "C. Sitawarin", "D. Wagner"],
       "year": 2024,
       "arxiv_id": "2402.06363",
       "relevance": "Defense against prompt injection using structured query formatting to separate trusted and untrusted inputs."
     },
     {
       "title": "Pal: Proxy-guided black-box attack on large language models",
       "authors": ["C. Sitawarin", "N. Mu", "D. Wagner", "A. Araujo"],
       "year": 2024,
       "arxiv_id": "2402.09674",
       "relevance": "Proxy-guided black-box attack using logprobs; surveys API access levels across vendors (Table 6 referenced by fun-tuning)."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Security researchers and red teamers can adapt the technique, though the specific Gemini exploit was mitigated by Google in April 2025."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the assumption that closed-weight models are safe from optimization-based attacks by revealing that fine-tuning APIs leak sufficient loss information."
     },
     "fear_safety": {
       "score": 3,
       "justification": "Demonstrates a novel, practical attack surface on proprietary LLMs that exploits a fundamental utility-security tradeoff in fine-tuning interfaces."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Responsible disclosure to Google was handled cooperatively; no major controversy beyond the technical vulnerability itself."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub, but the specific Gemini exploit has been patched and requires API access to test."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Targets Google Gemini (major product), published at IEEE S&P (top security venue), from UCSD researchers."
     }
   }
 }