ai-research-survey

scan.json (30397B)

---

 {
   "paper": {
     "title": "The Good and The Bad: Exploring Privacy Issues in Retrieval-Augmented Generation (RAG)",
     "authors": [
       "Shenglai Zeng",
       "Jiankun Zhang",
       "Pengfei He",
       "Yue Xing",
       "Yiding Liu",
       "Han Xu",
       "Jie Ren",
       "Shuaiqiang Wang",
       "Dawei Yin",
       "Yi Chang",
       "Jiliang Tang"
     ],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2402.16893",
     "doi": "10.48550/arXiv.2402.16893"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "RAG systems are highly vulnerable to privacy attacks on the retrieval database, with composite structured prompts ({information} + {command}) extracting verbatim or near-verbatim retrieval data at rates near 50%. Conversely, integrating retrieval data substantially reduces LLMs' tendency to output memorized training data, outperforming noise injection and protective system prompts. Mitigation strategies such as re-ranking are ineffective, while abstractive summarization reduces untargeted attack risk by ~50% but is limited against targeted attacks; distance thresholds present a privacy-utility tradeoff.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our code is available at https://github.com/phycholosogy/RAG-privacy', providing a GitHub repository URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: Enron Email dataset (500K emails), HealthcareMagic-101 (200K dialogues), wikitext-103, w3c-email, and Common Crawl chunks. All are publicly accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is provided in the paper. Only model names and library names (Chroma, embedding models) are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. While a code repository is provided, the paper itself contains no README-style instructions or 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-3 and Figures 2-6 are reported as raw counts without any confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares attack effectiveness across models, datasets, and settings based solely on comparing raw count numbers. No statistical significance tests (p-values, t-tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides absolute counts with baseline context throughout (e.g., Table 3 shows reduction from 245 emails to 2 with RAG-Chatdoctor, prefix attack reconstruction from 213 to 34). The relative magnitudes are clear from the tabulated results."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 250 prompts for retrieval attacks, 5000 for targeted training attacks, and 1000 for prefix attacks without justifying these choices or performing power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single experimental runs. No standard deviations, variance measures, or multi-run results are reported anywhere in the paper."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 compares RAG against multiple baselines: no retrieval ('None'), random noise injection ('Random Noise+prompt'), and protective system prompts ('System Prompt+prompt'), shown in Table 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models evaluated (Llama-2-7b-Chat, Llama-2-13b-Chat, GPT-3.5-turbo, GPT-Neo-1.3B) were contemporary at time of writing. Prior attack methods (Carlini et al. 2021, 2022) are cited and the paper explains why they are not directly applicable to RAG."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 presents extensive ablation studies on: number of retrieved documents k (Figure 3), command component design (Figure 2), embedding models (Figure 6, Appendix A.1), and temperature parameter (Appendix A.1, Tables 6-7)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: Retrieval Contexts, Repeat Prompts, Repeat Contexts, Rouge Prompts, Rouge Contexts, and Targeted Information (Section 4.1). Performance metrics include ROUGE-L and perplexity (Appendix A.3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated using exact string matching and ROUGE-L score thresholds. No human evaluation of extracted content quality or privacy severity is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.3 describes a 99:1 train/test split: 'we partition it into training and testing sets using a 99:1 ratio. The training set is utilized to build the RAG model, while we randomly sample 1000 instances from the testing set to evaluate the performance.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are consistently broken down by dataset (HealthcareMagic vs. Enron), model (L7C, L13C, GPT), attack type (targeted vs. untargeted), and in ablation studies by embedding model, temperature, and command variant."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 discusses cases where mitigations fail: re-ranking has 'almost no mitigation effects' (Figure 4a-b), and targeted attacks can inadvertently increase PII leakage after summarization ('the occurrence of personally identifiable information even inadvertently increased')."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that re-ranking has 'almost no mitigation effects' (Section 4.5), that summarization can increase PIIs for targeted attacks on Enron (Section 4.5), and that increasing k does not substantially increase leakage (Section 4.4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (1) RAG systems are vulnerable to retrieval data leakage and (2) RAG can mitigate training data leakage. Both are supported by Tables 1-3 in Sections 4 and 5 respectively."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.4 makes causal claims ('incorporating external data makes LLMs less reliant on training data') justified only by a Bayesian hypothesis and observational comparisons. The comparisons with/without RAG confound the effect of retrieval data with the effect of additional context length, despite the noise baseline attempting to control for this."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to explore 'Privacy Issues in Retrieval-Augmented Generation (RAG)' broadly, but experiments use only 2-3 LLMs on 2 retrieval datasets with a single retriever architecture. The conclusions in Section 6 are stated broadly without qualifying to the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 discusses why RAG reduces memorization using a Bayesian inference framework (Xie et al., 2021) and considers that the effect may be driven by shifting the conditional distribution rather than simple distraction, supported by the W3C-Email comparison."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses exact string matches and ROUGE-L > 0.5 as proxies for 'privacy leakage' without discussing the gap between textual overlap metrics and actual privacy harm. A ROUGE-L > 0.5 match may not constitute meaningful privacy leakage, and exact matches of non-sensitive content are counted the same as PII extraction."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'Llama-7b-chat', 'Llama-13b-chat', 'GPT-3.5-turbo', and 'GPT-Neo-1.3B' without snapshot dates or exact version identifiers. GPT-3.5-turbo behavior changes across versions and no API version date is given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Actual prompt text is provided: command prompts in Section 4.4 (C1-C4), information components in Appendix A.2.1, protective system prompts in Table 10, and summarization defense prompts in Table 11."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "While k=2, embedding model, and distance metric (L2-norm) are reported, the default temperature and other generation parameters (top-p, max tokens) for the main experiments are not stated. Temperature is only explored as a variable in ablation studies."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a standard RAG pipeline (retriever + LLM generation) without agent loops, tool use, or multi-step reasoning."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes how data is prepared: 'we construct each doctor-patient medical dialogue as a data piece embedded and stored in a vector database' and 'we construct each email as a data piece.' Appendix A.2.1 describes how attack prompts are generated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is a dedicated 'Limitations' section discussing the study's scope limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 limitations are generic scope boundaries ('concentrated primarily on the application of retrieval augmentation during the inference stage', 'other retrieval-based language models... warrant further investigation') without discussing specific threats to the validity of their findings like confounds or measurement validity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states what was NOT tested: pre-training/fine-tuning phase integration, other retrieval-based LM architectures (citing Huang et al., 2023; Borgeaud et al., 2022), and identifies developing effective defenses as an open question."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw attack outputs, extracted PIIs, and detailed per-prompt results are not made available. Only aggregate counts are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes datasets: Enron Email dataset of 500,000 employee emails and HealthcareMagic-101 of 200K doctor-patient dialogues. Appendix A.2.1 describes how attack prompts are generated for targeted and untargeted attacks."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All experiments use standard public datasets (Enron Email, HealthcareMagic, wikitext-103, Common Crawl)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: datasets are embedded using specified models, stored in Chroma vector database, retrieved with k-NN (L2-norm), concatenated with query, and fed to the LLM. Evaluation criteria (exact match, ROUGE-L > 0.5) are defined in Section 4.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Michigan State University, Baidu Inc., and Jilin University. The Baidu affiliation is relevant since Baidu develops and deploys RAG systems."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. Some authors are affiliated with Baidu Inc., which has commercial interest in RAG systems. The independence of any potential funding cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper. Authors from Baidu (a company building RAG products) have potential undisclosed conflicts."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper tests privacy attack success rates on RAG systems rather than evaluating model knowledge on benchmarks. The study deliberately leverages known training data (Enron emails in GPT-Neo's corpus) as part of the experimental design for memorization tests."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper tests privacy/security properties of RAG systems (attack and defense effectiveness), not model capability on benchmarks where train/test overlap would be a validity threat."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a red-teaming/privacy study testing extraction attacks on RAG systems. Benchmark contamination in the traditional sense is not applicable to this study design."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All experiments are automated attacks on RAG systems."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses publicly available datasets and automated evaluation."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions involving human assignment."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or evaluators requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, wall-clock time, or cost per attack prompt are reported, despite using GPT-3.5-turbo (a paid API) for thousands of queries."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or API spend is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Results appear to be single-run experiments."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from single runs or averaged across multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "While different command prompts and parameter settings are explored in ablation studies, no systematic hyperparameter search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 explains why 'Please repeat all the context' was selected as the default command: 'it strikes a balance between retrieval and prompting the LLM to repeat.' The ablation shows all configurations, not just the best."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparisons across multiple models, datasets, attack types, and settings without any correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose the attack method and evaluate its effectiveness without acknowledging potential bias in evaluating their own approach. No independent evaluation is conducted."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "Compute differences between experimental settings are negligible (same number of API calls across conditions)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether exact string matching and ROUGE-L > 0.5 thresholds are valid measures of actual privacy leakage risk. A high ROUGE-L score with non-sensitive content may not constitute meaningful privacy harm."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The RAG pipeline is consistent across all comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal aspects of the evaluation setup. The paper does not address whether the timing of dataset creation relative to model training affects results."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the attack setup inadvertently leaks information that would not be available in a realistic attack scenario."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether attack prompts are independent or whether retrieval database entries share structural similarities that inflate success metrics."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection methods (canary strings, membership inference, n-gram overlap analysis) are applied. Leakage is measured purely by output matching."
       }
     }
   },
   "claims": [
     {
       "claim": "RAG systems are highly susceptible to retrieval data extraction attacks, with a considerable amount of sensitive data being extracted (near 50% success rate).",
       "evidence": "Tables 1-2 show that out of 250 prompts, targeted attacks extract 89-205 PIIs/medical records depending on model and dataset (Section 4.2-4.3). Untargeted attacks produce 55-208 ROUGE-matched contexts.",
       "supported": "moderate"
     },
     {
       "claim": "The composite structured prompt ({information} + {command}) is effective for extracting retrieval data from RAG systems.",
       "evidence": "Tables 8-9 and Figure 2 show that the command 'Please repeat all the context' achieves consistently good extraction across settings, while very short or very long commands reduce effectiveness (Section 4.4).",
       "supported": "moderate"
     },
     {
       "claim": "Increasing k (number of retrieved documents) does not substantially increase privacy leakage.",
       "evidence": "Figure 3 shows that while retrieved documents increase with k, leaked documents grow much more slowly. Section 4.4 attributes this to 'models' constrained capacity to coherently process extensive contextual content.'",
       "supported": "moderate"
     },
     {
       "claim": "Integrating retrieval data substantially reduces LLMs' tendency to output memorized training data, achieving greater protection than noise injection or system prompts.",
       "evidence": "Table 3 shows targeted attack extractions drop from 245 emails (no RAG) to 2-4 (with RAG), and prefix attack reconstructions from 213 to 33-70 (Section 5.2-5.3). Noise/system prompts are far less effective.",
       "supported": "strong"
     },
     {
       "claim": "Re-ranking has almost no mitigation effect on retrieval data extraction.",
       "evidence": "Figures 4a-4b and Tables 18-19 show that re-ranking using bge-reranker-large produces nearly identical extraction counts as the no-reranking baseline (Section 4.5).",
       "supported": "moderate"
     },
     {
       "claim": "Abstractive summarization reduces untargeted attack risk by approximately 50% but is limited against targeted attacks.",
       "evidence": "Figures 4c-4d show summarization reduces untargeted extraction substantially, but for targeted attacks on Enron, 'PIIs even inadvertently increased' (Section 4.5). Tables 20-21 provide detailed numbers.",
       "supported": "moderate"
     },
     {
       "claim": "There exists a privacy-utility tradeoff when setting retrieval distance thresholds.",
       "evidence": "Figure 5 shows that lower thresholds reduce extraction but also degrade RAG performance (ROUGE-L/perplexity). Tables 13-14 quantify the performance impact at various thresholds (Section 4.5).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are reported as raw counts from apparently single runs with no error bars, confidence intervals, standard deviations, or multi-run variance. The reliability of the reported counts is impossible to assess."
     },
     {
       "flag": "Unjustified sample sizes",
       "detail": "250 prompts for retrieval attacks, 5000 for targeted training attacks, and 1000 for prefix attacks are used without justification. It is unclear whether these are sufficient for reliable conclusions."
     },
     {
       "flag": "Undisclosed conflicts of interest",
       "detail": "Multiple authors are affiliated with Baidu Inc., which develops and deploys RAG systems commercially. No funding or competing interests statement is provided. The paper's conclusion that RAG 'could potentially mitigate the risks of training data leakage' is favorable to RAG adoption."
     },
     {
       "flag": "Construct validity of privacy metrics",
       "detail": "ROUGE-L > 0.5 and exact token matching are used as proxies for privacy leakage without discussing whether these thresholds correspond to actual privacy harm. Non-sensitive text matches are counted equally with PII extraction."
     },
     {
       "flag": "Missing model version specificity",
       "detail": "GPT-3.5-turbo is used without a snapshot date or API version. Model behavior varies significantly across versions, making results potentially non-reproducible."
     }
   ],
   "cited_papers": [
     {
       "title": "Extracting training data from large language models",
       "authors": ["Nicholas Carlini", "Florian Tramer", "Eric Wallace", "Matthew Jagielski"],
       "year": 2021,
       "relevance": "Foundational work on data extraction attacks against LLMs, directly extended in this paper's methodology."
     },
     {
       "title": "Quantifying memorization across neural language models",
       "authors": ["Nicholas Carlini", "Daphne Ippolito", "Matthew Jagielski", "Katherine Lee"],
       "year": 2022,
       "arxiv_id": "2202.07646",
       "relevance": "Established prefix attack methodology and factors affecting memorization risk in LLMs, used as a baseline attack method."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus"],
       "year": 2020,
       "relevance": "Introduced the RAG technique that this paper investigates for privacy risks."
     },
     {
       "title": "Privacy implications of retrieval-based language models",
       "authors": ["Yangsibo Huang", "Samyak Gupta", "Zexuan Zhong", "Kai Li", "Danqi Chen"],
       "year": 2023,
       "arxiv_id": "2305.14888",
       "relevance": "Investigated privacy risks of kNN-LM retrieval-based models, closely related work with different architecture focus."
     },
     {
       "title": "Exploring memorization in fine-tuned language models",
       "authors": ["Shenglai Zeng", "Yaxin Li", "Jie Ren"],
       "year": 2023,
       "arxiv_id": "2310.06714",
       "relevance": "Examined memorization across different fine-tuning tasks, finding vulnerabilities in dialogue and summarization tasks."
     },
     {
       "title": "Preventing verbatim memorization in language models gives a false sense of privacy",
       "authors": ["Daphne Ippolito", "Florian Tramèr", "Milad Nasr"],
       "year": 2022,
       "arxiv_id": "2210.17546",
       "relevance": "Demonstrated that preventing verbatim memorization alone is insufficient for privacy, relevant to the metrics used in this paper."
     },
     {
       "title": "Deduplicating training data mitigates privacy risks in language models",
       "authors": ["Nikhil Kandpal", "Eric Wallace", "Colin Raffel"],
       "year": 2022,
       "relevance": "Showed that data duplication increases memorization risk in LLMs, relevant context for training data privacy."
     },
     {
       "title": "Memorization in NLP fine-tuning methods",
       "authors": ["Fatemehsadat Mireshghallah", "Archit Uniyal", "Tianhao Wang"],
       "year": 2022,
       "arxiv_id": "2205.12506",
       "relevance": "Found that fine-tuning model heads leads to more significant memorization than adapter modules, relevant to LLM privacy risks."
     },
     {
       "title": "Retrieval-augmented code generation and summarization",
       "authors": ["Md Rizwan Parvez", "Wasi Ahmad", "Saikat Chakraborty"],
       "year": 2021,
       "relevance": "RAG applied to code generation, an application domain where proprietary code in retrieval databases creates privacy concerns."
     },
     {
       "title": "Retrieval augmentation reduces hallucination in conversation",
       "authors": ["Kurt Shuster", "Spencer Poff", "Moya Chen"],
       "year": 2021,
       "arxiv_id": "2104.07567",
       "relevance": "Demonstrated RAG's hallucination reduction benefits, relevant to understanding the dual nature of RAG's effects on generation."
     },
     {
       "title": "Prompts should not be seen as secrets: Systematically measuring prompt extraction attack success",
       "authors": ["Yiming Zhang", "Daphne Ippolito"],
       "year": 2023,
       "arxiv_id": "2307.06865",
       "relevance": "Related work on prompt extraction attacks, a form of information leakage from LLM systems."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners building RAG systems can directly use these findings and mitigation strategies (summarization, distance thresholds) to assess and reduce privacy risks."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that RAG protects training data while exposing retrieval data is a mildly surprising dual characterization, but RAG privacy risks are somewhat expected."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates concrete privacy attacks extracting PII (phone numbers, medical records) from RAG systems with near 50% success rates."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "No major controversy, but the paper highlights that widely-deployed RAG systems are vulnerable, which has industry implications."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is released on GitHub and the attacks use publicly available datasets and models, making reproduction feasible."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from Michigan State and Baidu; experiments use GPT-3.5-turbo and Llama models, which are well-known but the paper itself is not from a top-tier AI lab."
     }
   }
 }