ai-research-survey

scan.json (29174B)

---

 {
   "paper": {
     "title": "SALAD: Systematic Assessment of Machine Unlearning on LLM-Aided Hardware Design",
     "authors": [
       "Zeng Wang",
       "Minghao Shao",
       "Rupesh Raj Karn",
       "Likhitha Mankali",
       "Jitendra Bhandari",
       "Ramesh Karri",
       "Ozgur Sinanoglu",
       "Muhammad Shafique",
       "Johann Knechtel"
     ],
     "year": 2025,
     "venue": "Workshop on Machine Learning for CAD (MLCAD)",
     "arxiv_id": "2506.02089",
     "doi": "10.1109/MLCAD65511.2025.11189152"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "The paper evaluates six machine unlearning algorithms (GA, GD, PO, NPO, RMU, SimNPO) on LLaMA 3.1-8B fine-tuned for Verilog generation across four threat scenarios: benchmark contamination, custom design withdrawal, malicious code removal, and IP leakage prevention. RMU and SimNPO provide the best balance between forgetting effectiveness and utility preservation, while gradient-based methods (GA, GD) achieve more complete erasure but severely degrade downstream RTL generation quality. Two to three unlearning epochs are generally sufficient for convergence, and different threat scenarios favor different algorithm choices.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/DfX-NYUAD/SALAD, mentioned in the abstract and detailed in Appendix B (Artifact Appendix)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The retain dataset (RTL-Coder training data) and most forget datasets are from public sources (VerilogEval, RTLLM, RTL-Repo, RTL-Breaker). The artifact appendix claims to provide retain/forget dataset splits and sensitive model links. However, the 703 in-house IP designs are proprietary and the artifact substitutes VeriLeaky examples (2 samples) for them."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Artifact appendix specifies: Python 3.11, Hugging Face Transformers 4.45, CUDA 12.2, 1-2×A100 (40/80 GB). Installation instructions reference the repository README."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Artifact appendix provides workflow: 'Run salad_unlearn.sh on the provided sensitive models to produce the unlearned model. Then run salad_eval.sh to evaluate unlearning performance.' README referenced for environment setup."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Figures 3-7 and Tables I-III report only point estimates (e.g., FR percentages, Min-K%++ values, Pass@K scores). No confidence intervals, error bars, or uncertainty quantification are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 6 unlearning algorithms across 4 scenarios and 3 epochs with claims like 'RMU and SimNPO strike the most effective balance' but uses no statistical significance tests. All comparisons are based on raw metric values."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context throughout, e.g., 'reducing FR from 65.1% to 1.0-1.9% at EP3' (Sec. V-B), 'reducing Min-K%++ from 85.1% to 30.8%' (Sec. X). Both original and unlearned values are consistently provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Dataset sizes are stated (156 VerilogEval, 50 RTLLM, 1134 custom, 703 IP, 835 malicious) but no justification is given for why these sizes are adequate for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs across unlearning epochs EP1-EP3. No standard deviations, variance across random seeds, or any spread measures are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six unlearning algorithms are compared (GA, GD, PO, NPO, RMU, SimNPO). Both 'Sensitive' (contaminated) and 'Clean' (uncontaminated) model baselines are included in Table I and Table III."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The unlearning methods are from 2024: TOFU framework [44], NPO [45], SimNPO [46], RMU [47]. These represent the current state of the art in LLM unlearning."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates existing unlearning algorithms rather than proposing a novel multi-component system. There are no components to ablate — the contribution is the systematic assessment framework, not a new algorithm."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: FR (Forget ROUGE), FP (Forget Probability), Min-K%, Min-K%++, PrivLeak for unlearning effectiveness, and Pass@K for downstream Verilog generation quality (Table I)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated via metrics (FR, Min-K%++, Pass@K, etc.). No human evaluation of generated Verilog quality or unlearning effectiveness is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Cross-contamination evaluation in Table I: models contaminated on RTLLM are evaluated on VerilogEval and vice versa. The retain/forget dataset split provides clear separation. Holdout datasets are explicitly mentioned for evaluating Verilog generation quality."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by threat scenario (Secs. V-VIII), by algorithm (6 methods), by epoch (EP1-EP3), and by metric. Table I provides per-benchmark Pass@K results. Table III breaks down by dataset type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failures are discussed throughout: GA/GD causing 'functional collapse' (Sec. V-B), PO showing instability with Min-K%++ 'spiking memorization to 99.9% at EP1' (Sec. V-B), NPO yielding 'inconsistent results' and 'reversible forgetting' (Sec. VII-B), SimNPO's 'Min-K%++ rebound to 36.4%' for IP protection (Sec. VIII-B)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results reported: GA 'completely failed across all metrics' on RTLLM (Table I), GD showed 'severe degradation' with Pass@15 dropping to 21, PO demonstrated 'instability,' NPO showed 'reversible forgetting' on malicious code. GA and GD consistently degrade downstream utility."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims machine unlearning enables 'selective removal of contaminated benchmarks, sensitive IP and design artifacts, or malicious code patterns' without full retraining. All four use cases (Secs. V-VIII) demonstrate this, with detailed quantitative results. The claim of 'effectively reducing data security risks' is supported by FR and Min-K%++ reductions."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'selective unlearning can improve RTL code generation by removing harmful memorization' (Sec. V-B) are supported by controlled experiments: each unlearning algorithm is applied to the same contaminated model with results compared across algorithms and epochs. This single-variable manipulation is adequate for the claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim applicability to 'LLM-Aided Hardware Design' broadly, but all experiments use a single model (LLaMA 3.1-8B) with specific RTL datasets. The conclusion acknowledges 'future work includes... exploring unlearning on reasoning-focused LLMs' but does not bound the main claims to the tested model."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why certain unlearning methods outperform others, or whether factors like dataset size, model architecture, or training dynamics could explain the observed differences. No robustness checks or confound analysis is provided."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures FR and Min-K%++ as proxies for security and claims to 'restore security and trust in LLM-aided hardware design' (Sec. I). The gap between metric reductions and actual security guarantees is not discussed — reducing FR to 8.5% does not necessarily mean the IP is fully protected."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The base model is specified as 'LLaMA 3.1-8B' (Sec. IV), which identifies a specific model version and size."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper performs inference for Verilog generation (Pass@K evaluation) and evaluation but does not provide the actual prompts or input formats used for generation or the evaluation benchmarks' prompt templates."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Sec. IV states: 'Fine-tuning is performed for 3 epochs with a learning rate of 1e-5 using the Adam optimizer. For inference, we set the temperature to 0.8, top-p to 0.75, and maximum context length to 2048 tokens.' Unlearning is run for 3 epochs."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper fine-tunes and applies unlearning algorithms to a language model directly."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states datasets were 'combined with the RTL-Coder training dataset' and split into retain/forget sets, but does not describe how raw RTL designs were processed into model training format, tokenization details, or any filtering/cleaning steps."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion mentions future work ('designing unlearning algorithms for code generation, establishing stricter evaluation protocols, and exploring unlearning on reasoning-focused LLMs') but does not substantively discuss current limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. There is no analysis of how results might be affected by the choice of single base model, dataset sizes, or the artificial contamination setup vs. real-world contamination patterns."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No mention of limitations to LLaMA 3.1-8B, to Verilog (not other HDLs), to the specific dataset sizes, or to the artificial contamination setup."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The artifact provides scripts and model links, but 703 proprietary in-house IP designs (Use Case 4) are not available. The artifact substitutes '2 IP leakage examples from VeriLeaky' — a fundamentally different dataset than the 703 designs used in the paper's experiments. Raw experimental outputs (metric logs) are not directly provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sec. IV describes data sources with specific counts: 156 VerilogEval samples, 50 RTLLM challenges, 1,134 RTL-Repo test set samples from public GitHub, 703 in-house IP designs from 'years of applied research and multiple tapeouts [49]-[55],' and 835 poisoned designs from RTL-Breaker."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks and existing design collections."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes dataset composition and the retain/forget split but does not document the pipeline from raw designs to model training format. No filtering criteria, removal counts, or intermediate processing steps are described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are mentioned anywhere in the paper. No acknowledgments section with grant information is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: NYU Tandon School of Engineering (USA) and NYU Abu Dhabi (UAE). The authors are not evaluating a commercial product they are affiliated with."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding disclosure is itself a gap."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses LLaMA 3.1-8B as the base model but does not state its training data cutoff date. This is relevant because the base model may already contain knowledge of VerilogEval/RTLLM benchmarks before fine-tuning."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is the paper's central topic. Table II explicitly measures contamination ratios using Min-K% and CDD metrics across VerilogEval, RTLLM, and custom designs under the clean model. The cross-contamination evaluation (Table I) tests models on benchmarks different from what they were contaminated with."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark contamination is the primary topic. Sec. V directly addresses contamination in VerilogEval and RTLLM, referencing prior work [3] on data contamination in foundational Verilog models. Table II quantifies contamination rates (up to 100% for RTLLM)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example cost is reported for either the fine-tuning, unlearning, or evaluation processes."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The artifact appendix mentions '1-2×A100 (40/80 GB)' hardware but does not quantify total GPU hours, training time, or computational budget for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be from single runs across unlearning epochs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs producing each result is never stated. Results are presented without indicating how many trials produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Fixed hyperparameters are used (lr=1e-5, temp=0.8, top-p=0.75) but no search budget or justification for these choices is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Table III and Sec. IX describe the selection criterion: 'We evaluate this impact using Euclidean distance with increased weighting on the FR to emphasize semantic forgetting.' The best algorithm and epoch per use case are selected based on proximity to clean model performance."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 6 algorithms across 4 scenarios and 3 epochs (72 comparisons) without any statistical tests at all, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all six unlearning methods themselves using the TOFU framework. No acknowledgment of potential bias from their own implementations of baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Different unlearning methods likely have different computational costs, but no performance-vs-compute analysis is provided. Methods are compared on effectiveness alone without considering relative computational expense."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "VerilogEval and RTLLM are used for downstream evaluation (Table I) without discussing whether Pass@K on these benchmarks adequately measures real-world RTL generation capability or unlearning effectiveness."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The paper evaluates direct model fine-tuning and unlearning without agentic frameworks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether LLaMA 3.1-8B's pre-training data already includes VerilogEval (2023) or RTLLM (2024) solutions, which would confound the contamination experiments."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information through input formatting or benchmark structure."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap or structural similarity between the retain dataset (RTL-Coder training data) and the forget/test datasets. The RTL-Coder data may share designs or patterns with VerilogEval/RTLLM."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Multiple leakage detection methods are applied: Min-K%, Min-K%++ (Sec. IV), and CDD (Table II). These are used to measure contamination levels in both clean and contaminated models."
       }
     }
   },
   "claims": [
     {
       "claim": "RMU and SimNPO provide the best balance between contamination removal and utility retention for benchmark decontamination.",
       "evidence": "Fig. 3 shows RMU reduces FR to 14.4% and Min-K%++ to 22.2% on RTLLM, while SimNPO reduces Min-K%++ from 85.1% to 30.8% on VerilogEval. Table I shows RMU achieves Pass@15 of 75 vs. 83 for clean models.",
       "supported": "moderate"
     },
     {
       "claim": "Gradient-based methods (GA, GD) achieve the most aggressive forgetting but severely degrade downstream utility.",
       "evidence": "Fig. 3 shows GA/GD reduce FR to 1.0-1.9% by EP3 on RTLLM. Table I shows GA completely fails on RTLLM benchmarks (all 1/0 scores) and GD drops Pass@15 to 21 on VerilogEval.",
       "supported": "strong"
     },
     {
       "claim": "Two to three unlearning epochs are sufficient for convergence across all threat scenarios.",
       "evidence": "Sec. V-B through VIII-B show most methods converge by EP2/EP3 across all four use cases. SimNPO and RMU show minimal drift after epoch 2.",
       "supported": "moderate"
     },
     {
       "claim": "Machine unlearning enables selective removal of malicious code patterns while preserving model utility.",
       "evidence": "Fig. 5 shows RMU reduces FR from 94.3% to 8.5% and Min-K%++ to 2.6% for malicious code. However, SimNPO only reduces FR to 67.5%, showing incomplete forgetting.",
       "supported": "moderate"
     },
     {
       "claim": "Selective unlearning can improve RTL code generation by removing harmful memorization.",
       "evidence": "Sec. V-B notes 'NPO and RMU even outperformed the contaminated model's original Pass@1 on VerilogEval' after unlearning RTLLM contamination. Table I shows RMU Pass@15=75 at EP3 vs. Sensitive=82.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical testing despite extensive comparisons",
       "detail": "The paper compares 6 algorithms × 4 scenarios × 3 epochs across multiple metrics but never applies statistical significance tests. All claims of superiority are based on comparing raw metric values. With this many comparisons, some observed differences could easily be noise."
     },
     {
       "flag": "Single model tested, broad generalization claimed",
       "detail": "All experiments use only LLaMA 3.1-8B, yet the paper claims results for 'LLM-aided hardware design' generally. No evidence that findings transfer to other model architectures or sizes."
     },
     {
       "flag": "No variance or reproducibility information",
       "detail": "All results appear to be single-run values. Without variance across seeds or runs, the stability of the reported metrics cannot be assessed. This is particularly concerning for unlearning methods where convergence behavior may be stochastic."
     },
     {
       "flag": "Artifact does not match paper experiments for IP protection",
       "detail": "The paper uses 703 proprietary in-house IP designs for Use Case 4, but the artifact provides only '2 IP leakage examples from VeriLeaky' — a fundamentally different and much smaller dataset. The main IP protection results cannot be independently verified."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations, threats to validity, or scope boundaries despite significant constraints (single model, artificial contamination setup, no real-world deployment evaluation)."
     }
   ],
   "cited_papers": [
     {
       "title": "Rtlcoder: Outperforming gpt-3.5 in design rtl generation with our open-source dataset and lightweight solution",
       "authors": ["S. Liu et al."],
       "year": 2024,
       "arxiv_id": "2312.08617",
       "relevance": "LLM fine-tuning for hardware code generation, demonstrates RTL-specific model training and serves as the retain dataset source for SALAD."
     },
     {
       "title": "Verigen: A large language model for verilog code generation",
       "authors": ["S. Thakur et al."],
       "year": 2023,
       "relevance": "Early work on LLM-based Verilog generation, demonstrating RTL code generation capabilities."
     },
     {
       "title": "Vericontaminated: Assessing llm-driven verilog coding for data contamination",
       "authors": ["Z. Wang et al."],
       "year": 2025,
       "arxiv_id": "2503.13572",
       "relevance": "Directly addresses data contamination in LLM Verilog generation, establishing the contamination problem that SALAD mitigates."
     },
     {
       "title": "Verileaky: Navigating ip protection vs utility in fine-tuning for llm-driven verilog coding",
       "authors": ["Z. Wang et al."],
       "year": 2025,
       "arxiv_id": "2503.13116",
       "relevance": "Explores IP protection and data extraction attacks on fine-tuned LLMs for Verilog, complementary defense approach to unlearning."
     },
     {
       "title": "Verilogeval: Evaluating large language models for verilog code generation",
       "authors": ["M. Liu", "N. Pinckney", "B. Khailany", "H. Ren"],
       "year": 2023,
       "relevance": "Key benchmark for evaluating LLM Verilog generation capability, used as both contamination target and evaluation benchmark in SALAD."
     },
     {
       "title": "Rtllm: An open-source benchmark for design rtl generation with large language model",
       "authors": ["Y. Lu", "S. Liu", "Q. Zhang", "Z. Xie"],
       "year": 2024,
       "relevance": "RTL generation benchmark used for contamination and evaluation in SALAD, shown to have 100% contamination rate under Min-K%."
     },
     {
       "title": "Chipnemo: Domain-adapted llms for chip design",
       "authors": ["M. Liu et al."],
       "year": 2023,
       "arxiv_id": "2311.00176",
       "relevance": "Industry approach to domain-adapting LLMs for chip design using proprietary data, relevant to IP leakage concerns."
     },
     {
       "title": "Tofu: A task of fictitious unlearning for llms",
       "authors": ["P. Maini", "Z. Feng", "A. Schwarzschild", "Z. C. Lipton", "J. Z. Kolter"],
       "year": 2024,
       "arxiv_id": "2401.06121",
       "relevance": "Framework for LLM unlearning evaluation that SALAD builds upon, implements GA, GD, PO unlearning methods."
     },
     {
       "title": "Rtl-breaker: Assessing the security of llms against backdoor attacks on hdl code generation",
       "authors": ["L. L. Mankali et al."],
       "year": 2024,
       "arxiv_id": "2411.17569",
       "relevance": "Demonstrates backdoor injection into LLMs for hardware design, source of malicious code patterns used in SALAD's Use Case 3."
     },
     {
       "title": "Asleep at the keyboard? Assessing the security of github copilot's code contributions",
       "authors": ["H. Pearce et al."],
       "year": 2025,
       "relevance": "Security assessment of LLM-generated code (Copilot), establishing the security risks of LLM code generation that motivate unlearning approaches."
     },
     {
       "title": "Rethinking machine unlearning for large language models",
       "authors": ["S. Liu et al."],
       "year": 2025,
       "relevance": "Comprehensive survey of machine unlearning methods for LLMs, provides theoretical foundation for the unlearning approaches evaluated."
     },
     {
       "title": "Detecting pretraining data from large language models",
       "authors": ["W. Shi et al."],
       "year": 2023,
       "arxiv_id": "2310.16789",
       "relevance": "Introduces Min-K% metric for detecting pre-training data in LLMs, used as a core evaluation metric in SALAD."
     }
   ]
 }