ai-research-survey

scan.json (30061B)

---

 {
   "paper": {
     "title": "Large Language Model Unlearning for Source Code",
     "authors": [
       "Xue Jiang",
       "Yihong Dong",
       "Huangzhao Zhang",
       "Tangxinyu Wang",
       "Zheng Fang",
       "Yingwei Ma",
       "Rongyu Cao",
       "Binhua Li",
       "Zhi Jin",
       "Wenpin Jiao",
       "Yongbin Li",
       "Ge Li"
     ],
     "year": 2025,
     "venue": "AAAI 2026",
     "arxiv_id": "2506.17125",
     "doi": "10.48550/arXiv.2506.17125"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PROD, a token-level distribution manipulation approach for LLM code unlearning, achieves superior trade-offs between forget quality and model utility compared to GA, DPO, NPO, and FLAT across three code unlearning tasks (copyrighted code, insecure code, deprecated APIs). Existing unlearning methods cause severe utility degradation on code due to the rigid syntax of programming languages, while PROD preserves code generation capability. PROD also demonstrates robustness against prefix injection adversarial attacks and generalizes across four 7B-parameter code LLMs.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our source code and data are available at https://github.com/jiangxxxue/PROD' in the Introduction section."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims data is available at the GitHub URL. The benchmark also uses publicly available datasets: Stack corpus, CyberSecEval, and VersiCode."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '4 NVIDIA A100 GPUs' and optimizer details (AdamW, weight decay 0.01) but provides no software environment specification (Python version, library versions, requirements.txt, or Dockerfile)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. A GitHub link is given but the paper itself contains no README-style reproduction guide or commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The main results in Table 1 are point estimates (PDR percentages) with no confidence intervals or ± notation. Figure 5 shows min/max ranges for adversarial attacks only, but the primary results lack uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims like 'PROD achieves superior overall performance' and 'significantly outperforms' are based on comparing raw numbers without any formal tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 'an average relative gain of 124% over the strongest competitor' for PDR. Table 1 provides absolute PDR values for all methods across tasks (e.g., PROD 41.8% vs GA 15.3% on copyright), giving baseline context for understanding effect magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes: 100 copyrighted files, 1,916 insecure snippets, 3,449 deprecated API snippets, 5 random seeds, or 3 human evaluators. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'unlearning training is conducted five times using different random seeds, and the final evaluation results are averaged across the five runs' but does not report standard deviation, variance, or any spread measure in Table 1 or elsewhere in the main results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four baselines are compared: Gradient Ascent (GA), Direct Preference Optimization (DPO), Negative Preference Optimization (NPO), and Forget Data Only Loss Adjustment (FLAT)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are from 2023-2025: GA (Yao et al. 2024), DPO (Rafailov et al. 2023), NPO (Zhang et al. 2024), FLAT (Wang et al. 2025). The paper describes them as spanning 'current LLM unlearning paradigms.'"
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 presents ablation studies: (a) alternative loss functions (KL, JS vs cross-entropy), (b) hyperparameter p for noise elimination (0.2, 0.4, 0.8, 0.9, 1.0), and (c) hyperparameter α for forget strength (-0.125 to 1)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: forget quality (1-BLEU for copyright, combined 1-BLEU + security pass rate for insecurity, exact-match accuracy for deprecated API), model utility (HumanEval functional correctness), and the composite PDR metric."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "A perceptual quality evaluation is conducted with 'three independent volunteer evaluators (each with two years or more of software development experience)' on 20 samples per task, plus GPT-4 evaluation on all samples. Results reported in Table 2."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Forget quality is measured on held-out portions of forget data (e.g., second half of copyrighted files). Model utility is measured on HumanEval, which is separate from the training/unlearning data. Deprecated API task uses a temporal split with deprecation boundary."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down PDR per task (copyright, insecurity, deprecation). Figure 3 shows per-task forget quality vs model utility curves. Figure 4 shows per-model results across 4 LLMs."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses failure modes of baselines in detail (mute refusal, token collapse, syntactic incoherence in Figure 1) but does not show or discuss failure cases of PROD itself. No qualitative examples where PROD fails are provided."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study reports configurations that don't work well: KL and JS divergence underperform cross-entropy (Figure 6a), no noise elimination yields worst results (Figure 6b), and certain α values degrade performance (Figure 6c). The paper also notes 'short forget sequences yf often require more training steps since the supervisory signal is weak.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'superior overall performance' are supported by Table 1 (highest PDR on all tasks). 'Broad applicability' is supported by Figure 4 (4 LLMs). 'Superior robustness against adversarial attacks' is supported by Figure 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (removing noise elimination, changing loss functions, varying hyperparameters) which are controlled single-variable manipulations. The claim that PROD's token-level manipulation preserves utility is supported by the mechanism design and empirical ablations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Large Language Model Unlearning for Source Code' broadly, but all tested models are approximately 7B parameters (CodeLlama-7B, Qwen2.5-Coder-7B, Deepseek-coder-6.7B, Starcoder-7B). No larger or smaller models are tested. No discussion of whether results generalize beyond ~7B models or beyond the three specific tasks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for PROD's superior performance. For example, the distribution sculpting could benefit from the specific memorization training setup, or the improvements could be task/scale-specific. No confound analysis is provided."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements. For copyright, they explicitly justify BLEU as a proxy: 'the law of copyright protects expression instead of ideas, textual similarity is employed as an indicator of potential copyright infringement.' Forget quality and model utility are directly measured, not framed as broader constructs."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are given: CodeLlama-7B, Qwen2.5-Coder-7B, Deepseek-coder-6.7B, Starcoder-7B. These are specific open-source models identifiable by name and size."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The GPT-4 evaluation prompt used for the perceptual quality study is not provided. For the main evaluation, code prefixes serve as inputs, but only one example is shown in Figure 1. The actual evaluation inputs (100 copyrighted files, CyberSecEval prompts) are not included in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive hyperparameters reported: learning rate grid {1e-4 to 1e-6}, batch size 32, 10 epochs, AdamW with weight decay 0.01, max sequence length 1024, greedy decoding (zero temperature), PROD p=0.8, α=0, NPO/DPO β=0.1."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. PROD is a direct model fine-tuning/post-training method."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Preprocessing is documented for each task: copyrighted code (random draw from Stack, filtering test files/toy examples/templates), insecure code (from CyberSecEval, 1,916 snippets with 50 CWE types), deprecated API (three steps: version filtration, file filtration, temporal split, resulting in 252 packages and 3,449 snippets). Memorization training uses 1:9 forget-to-pretrain ratio."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section exists. The paper has only a Conclusion section with no discussion of limitations or threats to validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. No mention of specific weaknesses of the evaluation approach."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what the results do NOT show, e.g., whether results apply to models of different sizes, other programming languages, or real-world memorization (vs. the artificial memorization training setup)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our source code and data are available at https://github.com/jiangxxxue/PROD.' The underlying benchmark datasets (Stack corpus, CyberSecEval, VersiCode) are publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described for each task: 100 files randomly drawn from Stack corpus with filtering criteria, 1,916 CyberSecEval snippets covering 50 CWE types across 8 languages, and VersiCode-derived deprecated API data with explicit preprocessing pipeline."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the human evaluation: 'three independent volunteer evaluators (each with two years or more of software development experience)' — no description of how they were recruited, where they are from, or whether the recruitment could introduce bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: for copyrighted code, Stack → random draw → filter non-expressive code → 100 files. For deprecated API, VersiCode → version filtration → file filtration (≥3 snippets) → temporal split → 252 packages, 3,449 snippets. Memorization training pipeline (1:9 ratio mixing) is also described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists: 'National Key R&D Program under Grant No. 2023YFB4503801, the National Natural Science Foundation of China under Grant No. 62192733, 62192730, 62192731, and the Major Program (JD) of Hubei Province (No.2023BAA024).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Peking University, Tongyi Lab (Alibaba Group), and Verdent AI. The note 'Work done during Xue Jiang and Yihong Dong's internship at Tongyi Lab' is also disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from Chinese government research programs (NSFC, National Key R&D Program, Hubei Province program), which have no financial stake in PROD outperforming baselines."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Some authors are from Alibaba's Tongyi Lab, which has commercial interests in code LLMs, but no declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four models (CodeLlama-7B, Qwen2.5-Coder-7B, Deepseek-coder-6.7B, Starcoder-7B)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval (used for model utility) appeared in any model's pre-training data. For the forget tasks, memorization is intentionally induced through training, but overlap between the Stack-sourced forget data and model pre-training is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021 and all four models were trained after 2021, meaning they likely encountered HumanEval during pre-training. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration mentioned for the human perceptual quality evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the study involving 3 human evaluators."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only minimal information: 'three independent volunteer evaluators (each with two years or more of software development experience).' No other demographics (education, role, programming languages known, etc.)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "The only criterion mentioned is 'two years or more of software development experience.' No formal inclusion/exclusion criteria or screening process described."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study with condition assignment — all evaluators assessed all samples in a comparative manner."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 'Each human evaluator assesses all samples without knowing which approach produced each sample.'"
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information about whether all 3 evaluators completed all evaluations, or any discussion of evaluator attrition or incomplete assessments."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or wall-clock time is reported for PROD or any baseline. The distribution sculpting step adds overhead that is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned ('4 NVIDIA A100 GPUs') but total GPU hours, training time, or total compute budget is not stated for any experiment."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'conducted five times using different random seeds, and the final evaluation results are averaged across the five runs' but does not report seed-to-seed variation or sensitivity analysis. Only averaged results are presented."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 'The unlearning training is conducted five times using different random seeds, and the final evaluation results are averaged across the five runs.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "The search procedure is described: 'we select the learning rate via grid search from the set {1e-4, 5e-5, 1e-5, 5e-6, 1e-6}, optimizing for the forget quality metric on forget data.' Search method (grid), search space (5 values), and selection criterion are stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Selection criterion is stated: learning rate selected by 'optimizing for the forget quality metric on forget data' via grid search. PROD hyperparameters (p=0.8, α=0) are justified by ablation results in Figure 6."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines and compare against their own system with no acknowledgment of author-evaluation bias. No independent evaluation or discussion of whether baseline reimplementations may underperform original implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison of compute budgets between PROD and baselines. PROD requires an additional distribution collection step that may increase compute, but this cost difference is not analyzed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval is used for model utility measurement without discussing whether it adequately captures general code generation capability. The paper briefly justifies BLEU for copyright ('the law protects expression') but does not discuss construct validity of its benchmarks more broadly."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. PROD is a direct model fine-tuning method with standard autocompletion inference."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The copyrighted code from Stack corpus and HumanEval for utility evaluation may have been in the models' pre-training data. Training cutoff dates are not stated."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "For the copyright task, the model is prompted with the first half of a file to see if it reproduces the second half. No discussion of whether this context provides sufficient information for correct completion regardless of memorization."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the Stack corpus files used as forget data overlap with the models' pre-training data (Stack is commonly used for training code LLMs). No verification of independence between forget set and pre-training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used (no canary strings, membership inference tests, n-gram overlap analysis, or decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "PROD achieves the best PDR score across all three unlearning tasks, with an average relative gain of 124% over the strongest competitor.",
       "evidence": "Table 1 shows PDR scores: PROD achieves 41.8% (copyright), 70.4% (insecurity), 58.0% (deprecation) vs best baselines of 15.3%, 24.5%, 51.6% respectively.",
       "supported": "strong"
     },
     {
       "claim": "Existing unlearning methods (GA, DPO, NPO, FLAT) cause severe model utility degradation on code generation, making models practically unusable.",
       "evidence": "Figure 3 shows all baselines' model utility drops to near-zero when achieving ~90% forget quality. Figure 1 demonstrates three failure modes: mute refusal, token collapse, and syntactic incoherence.",
       "supported": "strong"
     },
     {
       "claim": "PROD generalizes across four distinct code LLMs (CodeLlama-7B, Qwen2.5-Coder-7B, Deepseek-coder-6.7B, Starcoder-7B).",
       "evidence": "Figure 4 shows PROD achieves near-perfect forget quality while maintaining utility above baselines' best on all four LLMs on the copyrighted code task.",
       "supported": "moderate"
     },
     {
       "claim": "PROD is significantly more robust under prefix injection adversarial attack than all baselines.",
       "evidence": "Figure 5 shows PROD keeps similarity to copyrighted code below 0.05, while baselines exceed 0.3. PROD also shows smallest variance.",
       "supported": "moderate"
     },
     {
       "claim": "Human evaluators prefer PROD outputs over baselines with >70% win rate.",
       "evidence": "Table 2 shows win rates: 81% vs GA, 92% vs DPO, 76% vs NPO, 87% vs FLAT (human evaluation). GPT-4 evaluation confirms similar patterns.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Tiny human evaluation sample",
       "detail": "Only 3 human evaluators assessed 20 samples per task. This is too few evaluators to establish reliability of human judgments, and no inter-rater agreement statistics are reported."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations, threats to validity, or scope boundaries. This is a significant omission for a methods paper making broad claims."
     },
     {
       "flag": "All models are ~7B parameters",
       "detail": "All four tested models are approximately 7B parameters. No testing on larger or smaller models, yet the title and claims generalize to 'Large Language Models' broadly."
     },
     {
       "flag": "Artificial memorization setup",
       "detail": "For copyright and insecurity tasks, models are first trained to memorize the forget data before unlearning is applied. This may not reflect realistic scenarios where memorization occurs naturally during pre-training on web-scale data, potentially inflating PROD's advantage."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Comparative claims like 'significantly outperforms' and 'superior overall performance' are made without any statistical significance testing, despite running 5 seeds per experiment."
     },
     {
       "flag": "HumanEval contamination unaddressed",
       "detail": "HumanEval (published 2021) is used for model utility measurement, but all four models were trained after 2021 and may have seen HumanEval during pre-training. This confounds the utility measurement."
     }
   ],
   "cited_papers": [
     {
       "title": "Large Language Model Unlearning",
       "authors": ["Yuanshun Yao", "Xiaojun Xu", "Yang Liu"],
       "year": 2024,
       "arxiv_id": "2310.10683",
       "relevance": "Core LLM unlearning method (GA baseline) published at NeurIPS 2024; foundational to the field of LLM unlearning."
     },
     {
       "title": "Rethinking machine unlearning for large language models",
       "authors": ["Sijia Liu", "Yuanshun Yao", "Jinghan Jia"],
       "year": 2025,
       "relevance": "Nature Machine Intelligence review of LLM unlearning challenges and methods; establishes that traditional approaches are impractical for LLMs."
     },
     {
       "title": "Instruction tuning for secure code generation",
       "authors": ["Jingxuan He", "Mark Vero", "Gabriela Krasnopolska", "Martin Vechev"],
       "year": 2024,
       "relevance": "SaferCode approach for security-focused code LLM fine-tuning; directly relevant to secure code generation evaluation."
     },
     {
       "title": "Generalization or Memorization: Data Contamination and Trustworthy Evaluation for Large Language Models",
       "authors": ["Yihong Dong", "Xue Jiang", "Huanyu Liu"],
       "year": 2024,
       "relevance": "Studies data contamination in LLM evaluation, directly relevant to benchmark integrity and contamination concerns."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Primary target model for evaluation; widely-used open-source code LLM."
     },
     {
       "title": "LiCoEval: Evaluating LLMs on License Compliance in Code Generation",
       "authors": ["Weiwei Xu", "Kai Gao", "Hao He", "Minghui Zhou"],
       "year": 2025,
       "relevance": "Evaluates copyright/license compliance in LLM-generated code; directly relevant to the copyrighted code unlearning task."
     },
     {
       "title": "How and Why LLMs Use Deprecated APIs in Code Completion? An Empirical Study",
       "authors": ["Chong Wang", "Kaifeng Huang"],
       "year": 2024,
       "arxiv_id": "2406.09834",
       "relevance": "Studies deprecated API usage in LLM code completion; provides context for the deprecated API unlearning task."
     },
     {
       "title": "VersiCode: Towards version-controllable code generation",
       "authors": ["Tongtong Wu", "Weigang Wu"],
       "year": 2024,
       "arxiv_id": "2406.07411",
       "relevance": "Source dataset for the deprecated API unlearning benchmark; addresses version-specific code generation."
     },
     {
       "title": "Negative Preference Optimization: From Catastrophic Collapse to Effective Unlearning",
       "authors": ["Ruizhe Zhang", "Liwei Lin", "Yue Bai", "Song Mei"],
       "year": 2024,
       "relevance": "NPO baseline method that addresses GA's instability; key comparison point for LLM unlearning approaches."
     },
     {
       "title": "LLM Unlearning via Loss Adjustment with Only Forget Data",
       "authors": ["Yijiang Wang", "Jiaheng Wei"],
       "year": 2025,
       "relevance": "FLAT baseline method using f-divergence for unlearning; addresses the challenge of using only forget data."
     },
     {
       "title": "A Survey on Code Generation with LLM-based Agents",
       "authors": ["Yihong Dong", "Xue Jiang"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Survey of LLM-based code generation agents; provides broader context for code generation capabilities and challenges."
     },
     {
       "title": "Security Weaknesses of Copilot-Generated Code in GitHub Projects: An Empirical Study",
       "authors": ["Yujia Fu", "Peng Liang"],
       "year": 2025,
       "relevance": "Empirical study of security vulnerabilities in Copilot-generated code; relevant to the motivation for insecure code unlearning."
     }
   ]
 }