ai-research-survey

scan.json (27446B)

---

 {
   "paper": {
     "title": "LeakSealer: A Semisupervised Defense for LLMs Against Prompt Injection and Leakage Attacks",
     "authors": [
       "Francesco Panebianco",
       "Stefano Bonfanti",
       "Francesco Trovò",
       "Michele Carminati"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2508.00602",
     "doi": "10.48550/arXiv.2508.00602"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'LeakSealer's source code will be publicly available' (Section 5.1) — this is a promise of future release. No repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'We will open-source the dataset as a benchmark for future research' (Section 6.1). The PII dataset is only promised, not released. The OpenAI and ToxicChat datasets used are public, but the paper's novel contribution (PII dataset) is not yet available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is listed (Tesla P100 GPU, 16GB VRAM, Intel Xeon CPU) and fp16 precision is mentioned (Section 6), but no requirements.txt, Dockerfile, or library version specifications are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts for replicating results are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, and 3 report only point estimates (e.g., purity 0.97, AUPRC 0.97). No confidence intervals, error bars, or ± notation appear anywhere."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims LeakSealer 'outperforms' and 'significantly surpasses' baselines based solely on comparing metric values. No statistical significance tests (p-values, t-tests, etc.) are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results provide baseline context: e.g., 'AUPRC of 0.97' vs Llama Guard's 0.84 (Table 3), recall 0.88 vs second-best 0.65 (Table 2). Both method and baseline numbers are reported, allowing relative comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for dataset sizes. The PII dataset (N=1,048) and the use of existing datasets (N=1,680 and N=10,166) are not justified by power analysis or other means."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multiple-run results are reported. All results appear to be from single experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five baselines are compared: Llama Guard 3 8B, and LLM-As-A-Judge with DeepSeek-R1, GPT-4o, Ministral 8B, and Llama 3.1 8B (Section 6)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Llama Guard 3 (2024), DeepSeek-R1 (2025), GPT-4o (2024/2025), Ministral 8B (2024) — all recent and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "LeakSealer has multiple components (embedding, PCA, UMAP, HDBSCAN, classifier selection) but no ablation study is performed to measure individual component contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics reported: Accuracy, Precision, Recall, F1-score, AUPRC, and Purity (Tables 1-3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated using labeled datasets. The human labeling of the PII dataset is data annotation, not evaluation of the system's outputs. No human evaluation of LeakSealer's outputs is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Dynamic setting uses 'random 80%-20% split for the training and test sets' (Section 6.3). Static setting uses the ToxicChat test split (Appendix A)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by dataset (OpenAI, ToxicChat, PII) and by baseline method in Tables 1-3. Static vs dynamic settings also shown separately."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses where LeakSealer underperforms: on the smaller OpenAI dataset, GPT-4o and Llama Guard outperform it on several metrics. The paper attributes this to dataset size and potential pre-training bias in LLM baselines."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper honestly reports that on the OpenAI dataset, Llama Guard 3 achieves better AUPRC (0.90 vs 0.83) and GPT-4o achieves better recall (0.94 vs 0.75) in the dynamic setting (Table 3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of highest precision/recall on ToxicChat and AUPRC of 0.97 on PII dataset are supported by Tables 1-3. The claim about outperforming Llama Guard (0.84) is confirmed in Table 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims ('This shows the effectiveness of employing HDBSCAN's exemplars', Section 6.2) without ablation studies or controlled experiments to isolate what drives performance. The multi-component pipeline is not decomposed."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims defense 'Against Prompt Injection and Leakage Attacks' broadly, but testing is limited to toxicity detection and one specific PII leakage scenario. The paper acknowledges toxicity as 'a representative measure' in limitations but doesn't bound the generalization of the title claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses alternative explanations: LLM baselines may benefit from pre-training on public datasets (potential overfitting), dataset size may explain OpenAI performance differences, and class imbalance may inflate accuracy for some baselines."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges in limitations that 'toxicity of generated text was adopted as a representative measure for evaluating unsafe behaviors' — explicitly identifying the proxy gap between what is measured (toxicity detection) and the broader claim (defense against prompt injection)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models listed as 'Llama Guard 3 8B', 'Llama 3.1 8B', 'Ministral 8B', 'GPT-4o' (Section 6). GPT-4o has no snapshot date or API version. Llama models specify size (8B) but not specific checkpoint versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full system prompts for all LLM-As-A-Judge baselines are provided in Appendix D (Sections D.1-D.3). Email generation prompts are shown in Listing 1 (Appendix C)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Email generation hyperparameters reported (temperature 0.7, top_p 0.9, etc. in Appendix C). Embedding dimensions (PCA to 50, UMAP to 10) specified in Appendix B. However, classification model hyperparameters (SVM, RF, XGBoost, k-NN) and LLM inference settings (temperature for baselines) are not reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "LeakSealer is a clustering/classification pipeline, not an agentic scaffolding system. No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Preprocessing pipeline is documented in Appendix B: stella_en_400M_v5 embeddings (e=1024) → PCA to 50 dimensions → UMAP to 10 dimensions (static) or PCA only (dynamic) → HDBSCAN clustering. Data generation pipeline for PII dataset is detailed in Appendix C."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "A 'Limitations and Future Work' subsection exists in Section 8 (Conclusions) but contains only ~3 sentences, mostly forward-looking ('Future research directions include...'). This does not constitute substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The only specific limitation mentioned is using toxicity as a proxy for unsafe behavior. No discussion of threats such as: simulated (not real) human feedback, selection bias in the PII dataset, limited attack diversity, or generalizability to other domains."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No specific exclusions or untested scenarios are enumerated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The novel PII dataset is only promised for future release ('will be open-sourced'). No download link is provided. The OpenAI and ToxicChat datasets are publicly available but the paper's own data contribution is not."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "PII dataset creation is described in extensive detail in Section 6.1 and Appendix C: document generation methods (procedural, LLM-generated), identity generation via barnum library, manual review process, RAG system setup, and labeling procedure."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Data is synthetically generated and publicly available benchmarks are used."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: identity generation (barnum) → document creation (procedural/LLM) → manual review → RAG database population → question generation → LLM interaction → human labeling → dataset (N=1,048 interactions). Appendix C provides detailed steps for each document type."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section or acknowledgments mentioning grants or sponsors appear anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: three authors from Politecnico di Milano, one from ML cube (a company)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. One author is from ML cube (a company that may have commercial interest in LLM security solutions), but this is not discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. One author is affiliated with ML cube (a company), but no declaration of financial interests, patents, or equity is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests a defense framework (LeakSealer), not a pre-trained model's capability on benchmarks. The LLMs are used as baselines for comparison, but the core contribution is a semi-supervised clustering/classification approach."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper evaluates a defense system, not model knowledge. However, the paper does note potential contamination bias in LLM baselines (Section 7)."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "LeakSealer is a defense framework, not a pre-trained model evaluated on benchmark knowledge. Contamination does not apply to the primary contribution."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The PII dataset uses synthetic data with human labeling, but this is not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data is synthetic."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims LeakSealer is 'lightweight' and uses efficient ML architectures (Section 7, 'Performance remarks') but provides no specific latency, cost per example, or timing measurements."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (Tesla P100 GPU, 16GB VRAM) but total computational budget (GPU hours, training time, total API spend for baseline LLM evaluations) is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'nested cross-validation' for model selection among SVM, RF, XGBoost, and k-NN (Appendix B) but does not report the search budget, configurations tried, or compute spent on search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The classification model is 'robustly selected through a nested cross-validation procedure' among SVM, RF, XGBoost, and k-NN (Appendix B). Nested CV is a principled selection method that avoids overfitting to the test set."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons across 5 baselines, 3 datasets, and 2 settings are made with no correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system against baselines without acknowledging author-evaluation bias. No independent evaluation or discussion of this bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper qualitatively claims LeakSealer is lighter than LLM-based approaches (Section 7) but provides no quantitative compute comparison. LLM baselines require full inference vs lightweight ML classifiers, but this is not measured."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges in limitations that 'toxicity of generated text was adopted as a representative measure for evaluating unsafe behaviors' (Section 8), explicitly identifying the gap between the benchmark measure and the broader defense claim."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. LeakSealer is a clustering/classification pipeline, not an agentic system."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 and 7 discuss that 'LLMs are trained on web-crawled data, which raises concerns regarding potential overfitting due to the availability of certain datasets online.' The paper notes LeakSealer avoids this bias via its semi-supervised approach without pre-training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation features contain information not available at prediction time."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 80/20 random split for dynamic evaluation does not address whether train and test examples share structural similarities (e.g., similar attack patterns, same document types in PII dataset)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method (canary strings, membership inference, n-gram overlap) is applied. The contamination concern is raised conceptually but not tested."
       }
     }
   },
   "claims": [
     {
       "claim": "In the static setting, LeakSealer achieves the highest F1-score (0.77) on both ToxicChat and PII datasets, outperforming all baselines.",
       "evidence": "Table 2 shows F1 scores: LeakSealer 0.77 on ToxicChat (vs second-best 0.61) and 0.77 on PII (vs second-best 0.72 for GPT-4o).",
       "supported": "strong"
     },
     {
       "claim": "In the dynamic setting, LeakSealer achieves AUPRC of 0.97 on PII leakage detection, outperforming Llama Guard (0.84).",
       "evidence": "Table 3 and Figure 4c show AUPRC 0.97 for LeakSealer vs 0.84 for Llama Guard 3 on the PII dataset.",
       "supported": "moderate"
     },
     {
       "claim": "LeakSealer's recall for PII detection (0.88 static, 0.95 dynamic) significantly exceeds all baselines.",
       "evidence": "Table 2: recall 0.88 vs second-best 0.65 (Ministral). Table 3: recall 0.95 vs second-best 0.59 (GPT-4o and Ministral).",
       "supported": "moderate"
     },
     {
       "claim": "LeakSealer is not affected by pre-training bias that may inflate LLM baseline performance on public datasets.",
       "evidence": "Section 7 argues LeakSealer's semi-supervised pipeline avoids pre-training on web data. However, this is a plausibility argument without empirical verification.",
       "supported": "weak"
     },
     {
       "claim": "Cluster purity reaches 0.97 on ToxicChat, demonstrating effective separation of safe and unsafe interactions.",
       "evidence": "Table 1 reports purity 0.97 on ToxicChat. The paper notes this may partly reflect class imbalance (92% safe, 8% unsafe).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LeakSealer is a semisupervised framework combining clustering-based static analysis with HITL-informed dynamic defense for detecting prompt injection and PII leakage in LLM systems. On the PII leakage task, it achieves AUPRC of 0.97 vs 0.84 for Llama Guard, with notably higher recall (0.95 vs 0.31). Performance on ToxicChat is also superior (F1 0.72 vs 0.48 for Llama Guard). On the smaller OpenAI dataset, results are mixed, with Llama Guard achieving higher AUPRC (0.90 vs 0.83). The paper also introduces a curated PII leakage dataset (N=1,048) with labeled RAG interactions.",
   "red_flags": [
     {
       "flag": "No error bars or variance across runs",
       "detail": "All results are reported as point estimates from apparently single runs. No standard deviations, confidence intervals, or seed sensitivity analysis. The 80/20 random split could produce different results with different random seeds, but this is never tested."
     },
     {
       "flag": "Simulated human-in-the-loop",
       "detail": "The HITL pipeline uses ground truth labels from the dataset to simulate human feedback (Appendix B: 'the human feedback is simulated using the ground truth from the dataset'). Real-world HITL performance — where human annotators may disagree, make errors, or face ambiguous cases — is untested."
     },
     {
       "flag": "Code and data promised but not released",
       "detail": "Both LeakSealer source code and the PII dataset are described as 'will be' released. No repository URLs or download links are provided. Reproducibility cannot be verified."
     },
     {
       "flag": "Self-created benchmark evaluated by same team",
       "detail": "The PII dataset is created, labeled, and evaluated by the same research team with no independent validation. The dataset has not been previously published and thus could not introduce pre-training bias, but this also means no external quality check exists."
     }
   ],
   "cited_papers": [
     {
       "title": "Harnessing the Power of LLMs in Practice: A Survey on ChatGPT and Beyond",
       "authors": ["Jingfeng Yang", "Hongye Jin", "Ruixiang Tang"],
       "year": 2024,
       "relevance": "Survey on LLM capabilities and deployment patterns, relevant to understanding the landscape of LLM applications."
     },
     {
       "title": "Retrieval-augmented generation for large language models: A survey",
       "authors": ["Yunfan Gao", "Yun Xiong", "Xinyu Gao"],
       "year": 2023,
       "arxiv_id": "2312.10997",
       "relevance": "Survey on RAG systems, directly relevant to the RAG context vulnerability that LeakSealer addresses."
     },
     {
       "title": "Do Anything Now: Characterizing and Evaluating In-the-Wild Jailbreak Prompts on Large Language Models",
       "authors": ["Xinyue Shen", "Zeyuan Chen", "Michael Backes"],
       "year": 2024,
       "relevance": "Systematic analysis of jailbreak prompts in the wild, foundational work on the threat model LeakSealer defends against."
     },
     {
       "title": "The good and the bad: Exploring privacy issues in retrieval-augmented generation (RAG)",
       "authors": ["Shenglai Zeng", "Jiankun Zhang", "Pengfei He"],
       "year": 2024,
       "relevance": "Demonstrates PII leakage attacks against RAG systems, the specific threat scenario that motivates LeakSealer's PII defense."
     },
     {
       "title": "Llama guard: LLM-based input-output safeguard for human-ai conversations",
       "authors": ["Hakan Inan", "Kartikeya Upasani", "Jianfeng Chi"],
       "year": 2023,
       "arxiv_id": "2312.06674",
       "relevance": "Primary baseline for LeakSealer comparison; LLM-based safety classification approach that LeakSealer aims to outperform."
     },
     {
       "title": "An empirical study of llm-as-a-judge for llm evaluation: Fine-tuned judge models are task-specific classifiers",
       "authors": ["Hui Huang", "Yingqi Qu", "Jing Liu"],
       "year": 2024,
       "arxiv_id": "2403.02839",
       "relevance": "Evaluates LLM-as-a-judge approach used as a baseline in LeakSealer experiments."
     },
     {
       "title": "Prompt injection attacks and defenses in llm-integrated applications",
       "authors": ["Yupei Liu", "Yuqi Jia", "Runpeng Geng"],
       "year": 2023,
       "arxiv_id": "2310.12815",
       "relevance": "Framework for formalizing prompt injection attacks and benchmarking defenses, directly relevant to LLM security evaluation."
     },
     {
       "title": "ProPILE: Probing Privacy Leakage in Large Language Models",
       "authors": ["Siwon Kim", "Sangdoo Yun", "Hwaran Lee"],
       "year": 2024,
       "relevance": "Tool for probing PII leakage from LLM training data, addresses the pre-training leakage problem distinct from RAG leakage."
     },
     {
       "title": "Whispers in the Machine: Confidentiality in LLM-integrated Systems",
       "authors": ["Jonathan Evertz", "Merlin Chlosta", "Lea Schönherr"],
       "year": 2024,
       "arxiv_id": "2402.06922",
       "relevance": "Explores confidentiality issues in LLM systems using secret key extraction games, a related but distinct framing of the leakage problem."
     },
     {
       "title": "Defending Large Language Models Against Jailbreak Attacks via Layer-specific Editing",
       "authors": ["Wei Zhao", "Zhe Li", "Yige Li"],
       "year": 2024,
       "arxiv_id": "2405.18166",
       "relevance": "White-box jailbreak defense via model layer editing, contrasted with LeakSealer's model-agnostic approach."
     },
     {
       "title": "PARDEN, Can You Repeat That? Defending against Jailbreaks via Repetition",
       "authors": ["Ziyang Zhang", "Qizhen Zhang", "Jakob Foerster"],
       "year": 2024,
       "arxiv_id": "2405.07932",
       "relevance": "Repetition-based jailbreak defense that requires repeated inference, contrasting with LeakSealer's lightweight approach."
     },
     {
       "title": "ToxicChat: Unveiling Hidden Challenges of Toxicity Detection in Real-World User-AI Conversation",
       "authors": ["Zi Lin", "Zihan Wang", "Yongqi Tong"],
       "year": 2023,
       "relevance": "Benchmark dataset used in LeakSealer evaluation for toxicity detection in real-world AI conversations."
     }
   ]
 }