ai-research-survey

scan.json (31315B)

---

 {
   "paper": {
     "title": "Memorize or Generalize? Evaluating LLM Code Generation with Code Rewriting",
     "authors": [
       "Lizhe Zhang",
       "Wentao Chen",
       "Li Zhong",
       "Letian Peng",
       "Zilong Wang",
       "Jingbo Shang"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2503.02296",
     "doi": "10.48550/arXiv.2503.02296"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes code rewriting and the Memorization Risk Index (MRI) to distinguish harmful memorization from benign code reuse in LLM code generation. Key findings: (1) harmful memorization does not increase with model scale and often decreases, primarily by reducing accuracy drops under semantic shifts rather than reducing surface similarity; (2) memorization persists on harder tasks (BIGCODEBENCH) even at scale; (3) SFT improves accuracy but substantially inflates MRI, while PPO achieves a better risk-accuracy trade-off with lower memorization. Evaluated on MBPP+ and BIGCODEBENCH across Qwen-2.5 and Llama-3.1/4 model families.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The reproducibility statement says 'all evolved tasks and the prompts used during generation will be released publicly upon publication' — this is a promise of future release, not an actual release. No repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "While the base benchmarks MBPP+ and BIGCODEBENCH are public, the paper's primary contribution — the code rewriting, mutation, and paraphrase variant datasets — are not yet released. The reproducibility statement promises release 'upon publication.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2 mentions '4 NVIDIA A100 GPUs (80GB), using PyTorch and HuggingFace Transformers' but no specific library versions, requirements.txt, or Dockerfile are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The reproducibility statement acknowledges details are in appendices but no runnable scripts or commands are given."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2's mean ± SD values are computed across different models (footnote 3: 'unweighted arithmetic average computed per column across models'), not across experimental runs. No confidence intervals or error bars from repeated runs are reported for individual model evaluations."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'MRI falls from 0.0722 at 0.5B to 0.0113 at 14B') based solely on comparing point estimates. No statistical significance tests (p-values, t-tests, etc.) are used anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports absolute values with baselines (e.g., 'accuracy was boosted from 0.3158 → 0.3772' for SFT, 'MRI 0.0799 → 0.0795 (PPO) vs 0.1747 (SFT)'). Readers can assess magnitude from the reported numbers with baseline context throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges 'Due to the small size of MBPP+ test split (n = 78), estimation on this split may be imprecise and directional' (Section 4.1), but provides no formal justification for sample sizes or power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The ± SD values in Table 2 are cross-model variability, not run-to-run variance (footnote 3). Section 5.1.1's 'mean ± SD over 6 seeds' refers to 6 model sizes in the series, not random seeds. No multi-run variance for individual model evaluations is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares multiple model families (Qwen-2.5, Llama-3.1, Llama-4), includes mutation and paraphrase as reference baselines for robustness comparison (Section 3.2), and compares SFT vs PPO vs base models (Section 5.2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include Qwen-2.5 (2024), Llama-3.1 (2024), and Llama-4 (2024) — all very recent. GPT-5 (2025) is used for task generation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper decomposes MRI into Sim(Trew) and RADrew and analyzes each component independently. It compares three perturbation types (rewriting, mutation, paraphrase) and studies how each component behaves under different conditions (scale, fine-tuning strategy)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: MRI, Sim(Trew), RADrew, Acc(Tori), RADmut, RADpar. Table 2 reports all of these across models and benchmarks."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Two programmers manually reviewed 10% of generated perturbation tasks for quality (Section 3.1), but this validates the dataset, not the LLM outputs. The schema specifies 'manual classification of the benchmark or dataset itself does not count.' All LLM evaluation is automated (pass/fail on test suites)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'We use 4:1 train/test split for fine-tuning. For models that undergo SFT and PPO, we train on the training split and evaluate on the test split.' For instruction-only models, the complete benchmark is used (no training on it)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by benchmark (MBPP+ vs BIGCODEBENCH), model family (Qwen vs Llama), model size (0.5B to 32B/70B), perturbation type (rewriting, mutation, paraphrase), and training strategy (base, SFT, PPO)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides 5 regressed task examples per dataset (tasks that PASSED originally but FAILED under code rewriting) from Qwen2.5-Coder-32B-Instruct. The paper also discusses where memorization persists (harder tasks, BIGCODEBENCH) and where scaling fails to help."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that SFT substantially increases memorization (MRI 0.0799 → 0.1747), that scaling is non-monotonic on BIGCODEBENCH (MRI drops then rises at 32B for Qwen-Instruct), and that memorization persists on harder tasks despite scaling."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All three abstract claims are supported: (1) memorization doesn't increase with scale — Figure 2, Table 1; (2) SFT improves accuracy but introduces memorization — Figure 3; (3) PPO achieves better trade-off — Figure 3 shows lower MRI with maintained accuracy."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claims about SFT and PPO effects are supported by controlled experiments where the same base model (Qwen-2.5-7B) undergoes different training strategies. The paper uses ablation-style comparisons (base vs SFT vs PPO) with controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Evaluating LLM Code Generation' is broad. The abstract claims 'memorization does not increase with larger models' as a general finding, but the paper tests only 2 model families (Qwen, Llama) on 2 Python benchmarks. The scaling claim is not bounded to these specific families or Python tasks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes benign reuse from harmful memorization (Section 2.2), discusses that high similarity alone is not harmful if the model solves the perturbed task correctly, and considers task difficulty as a factor affecting results (MBPP+ vs BIGCODEBENCH). Section 5.1 discusses how surface-form reuse without failure does not constitute harmful memorization."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully defines what MRI measures (similarity × accuracy drop) and explicitly distinguishes this from what it claims to capture (harmful memorization). Section 1 defines harmful memorization behaviorally and Section 3.1.1 specifies exactly how MRI operationalizes this definition."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model checkpoints are named: 'Qwen-2.5-0.5B-Instruct' through '32B', 'Qwen-2.5-Coder-0.5B-Instruct' through '32B', 'Llama-3.1-8B-Instruct', 'Llama-3.1-70B-Instruct', 'Llama-4-Scout-17B-Instruct (16E)', 'Llama-4-Maverick-17B-Instruct (128E)'. GPT-5 version 'gpt-5-2025-08-07' specified in Appendix G."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts for code rewriting, mutation, paraphrase, and LLM judging are provided in Appendix B. The fine-tuning prompt template with exact text is provided in Appendix F (instruction_prefix, response_prefix, chat template)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Training hyperparameters are detailed in Appendix F (Tables 3-6). GPT-5 settings in Appendix G state 'temperature: default; top-p: default' which is not specifying actual values. Crucially, inference temperature/sampling settings for the evaluated Qwen and Llama models are not reported anywhere."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated directly on code generation tasks without multi-step workflows, tools, or agent frameworks."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The perturbation pipeline is documented: code rewriting process (Section 3.1), mutation noise budget formula (Appendix B.3), LLM judge validation (Section 3.1, Appendix B.2), manual review of 10% by two programmers, test case validation for rewritten tasks (Appendix B end). Train/test split (4:1) documented in Section 4.1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Section 6 'Conclusion and Future Works' mentions next steps (mitigation approach, evaluation transferability) but does not discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper acknowledges the small MBPP+ test split (n=78) may be 'imprecise and directional' (Section 4.1) but does not systematically discuss threats to validity such as GPT-5 task generation quality, benchmark representativeness, or whether code rewriting preserves difficulty."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion mentions 'while our current evaluation metrics are tailored for code generation' as a hint at scope, but no explicit statements about what the results do NOT show (e.g., only Python tasks, only two model families, only two benchmarks)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the evolved task datasets nor the model outputs are available. The reproducibility statement promises future release 'upon publication.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data generation process is described in detail: code rewriting pipeline (Section 3.1, Appendix B.1), mutation formula (Appendix B.3), paraphrasing process (Appendix B.4), GPT-5 configuration (Appendix G), validation pipeline including LLM judge and manual review."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (MBPP+, BIGCODEBENCH)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: original task → GPT-5 perturbation (with specific prompts in Appendix B) → LLM-as-judge validation (Appendix B.2) → manual review of 10% by two programmers → test case validation → 4:1 train/test split. The generation cost (~$450 per round) is stated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. There is no disclosure of any funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with University of California, San Diego. No conflicts with the evaluated models (Qwen, Llama, GPT-5 are from other organizations)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding disclosure does not confirm the absence of funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data cutoff dates are not stated for any of the evaluated models (Qwen-2.5, Llama-3.1, Llama-4), despite this being critical since the benchmarks MBPP+ and BIGCODEBENCH were published before these models were trained."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about detecting and measuring memorization of benchmark solutions by LLMs. Section 2.2 discusses prior work on memorization detection. The code rewriting method is explicitly designed to test whether models recall training solutions rather than generalize."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central contribution addresses benchmark contamination. Section 1 discusses that 'existing evaluations largely measure memorization via surface or structural overlap.' The code rewriting methodology is designed to reveal harmful memorization when models have seen benchmark solutions during training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on code generation benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Two programmers reviewed data quality but are not study participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example cost is reported for the model evaluations. Appendix G reports ~$450 for generating one round of perturbations via GPT-5 but this covers data generation, not the evaluation itself."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2: 'a total computational budget of approximately 40 GPU hours' on '4 NVIDIA A100 GPUs (80GB)'. Appendix G: '~450 USD' for task generation per round."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. Section 5.1.1's 'mean ± SD over 6 seeds' actually computes mean and SD across 6 model sizes in a series (0.5B through 32B), not random seeds — this is misleading terminology."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation runs per model is not stated. There is no indication of whether results are from single runs or averaged across multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Appendix F.1.1 states the learning rate was tuned 'with manual adjustments between 5×10⁻⁶ and 1×10⁻⁵ depending on model performance' but no search budget (configurations tried, compute spent on tuning) is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Checkpoint selection is clearly described: SFT uses 'the epoch immediately prior to overfitting' (Section 3.3.1, Table 5) based on validation loss. PPO uses 'the checkpoint with the highest test reward' (Section 3.3.2, Table 6). Both criteria are principled and stated in advance."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so there are no p-values to correct for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper proposes the MRI metric and evaluates it without acknowledging author-evaluation bias. No comparison against alternative memorization metrics (beyond surface similarity) is conducted, and no independent evaluation of MRI's validity is provided."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Performance is compared across model sizes (0.5B to 70B) without normalizing for compute budget. Footnote 2 notes Llama-4 MoE models have 'similar per-token activated compute' but no systematic compute-performance analysis is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses what MBPP+ (introductory) vs BIGCODEBENCH (harder) measure and how difficulty level affects memorization detection. Section 5.1 analyzes how benchmark difficulty interacts with MRI, and the paper's core contribution questions whether standard benchmarks actually measure generalization vs memorization."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in the evaluation. Models are evaluated directly on code generation tasks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper's entire framework addresses temporal leakage: code rewriting creates novel perturbations of existing benchmarks to test whether models are recalling training-era solutions. Section 2.2 discusses that models may have 'seen near-identical problems during pre-training.'"
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Feature leakage (whether the evaluation setup leaks answer information through context) is not discussed. The paper focuses on memorization of solutions but does not analyze whether the evaluation framework itself might provide hints."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the code rewriting variants are sufficiently independent from training data. The perturbations preserve surface syntax (by design), which could overlap with training data patterns."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "MRI itself is a leakage detection method: it combines similarity to original solutions with accuracy drop under semantic perturbation to identify when models are recalling training data rather than generalizing. This is a concrete, applied method (Section 3.1.1)."
       }
     }
   },
   "claims": [
     {
       "claim": "Memorization does not increase with larger models and in many cases alleviates as they scale.",
       "evidence": "Figure 2 shows MRI declining with scale for Qwen-2.5 Instruct on MBPP+ (0.0722 at 0.5B → 0.0000 at 32B). Table 1 shows similar trends for Llama families. On BIGCODEBENCH, the trend is milder and sometimes non-monotonic (Section 5.1).",
       "supported": "moderate"
     },
     {
       "claim": "SFT improves accuracy but substantially amplifies memorization.",
       "evidence": "Figure 3 and Section 5.2: Qwen-2.5-7B-SFT accuracy boosted from 0.3158 → 0.3772 on BIGCODEBENCH, but MRI increased from 0.0799 → 0.1747. Similar pattern for Coder variant (0.1392 → 0.1921).",
       "supported": "moderate"
     },
     {
       "claim": "PPO achieves a more balanced trade-off between memorization and generalization than SFT.",
       "evidence": "Section 5.2: PPO on Qwen-2.5-7B yields accuracy 0.3509 (vs 0.3772 SFT) with MRI 0.0795 (vs 0.1747 SFT). Similar for Coder: accuracy 0.3728 vs 0.4079, MRI 0.1336 vs 0.1921.",
       "supported": "moderate"
     },
     {
       "claim": "Memorization declines rapidly on simpler tasks but persists on more difficult ones.",
       "evidence": "Section 5.1.1: On MBPP+ (simpler), MRI reaches effectively zero at 32B scale. On BIGCODEBENCH (harder), MRI remains 0.1178 even for Qwen-2.5-Coder-32B-Instruct. RADrew on BIGCODEBENCH stays high (0.37-0.66) while near zero on MBPP+ for large models.",
       "supported": "strong"
     },
     {
       "claim": "Coder models encourage code reuse but do not substantially increase harmful memorization.",
       "evidence": "Section 5.1.1: Coder models have higher Sim(Trew) (0.3237 ± 0.0086 vs 0.2670 ± 0.0268 on BIGCODEBENCH) but comparable RADrew, resulting in only slight MRI increase (0.1367 ± 0.0224 vs 0.1142 ± 0.0247).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No run-to-run variance reported",
       "detail": "All results appear to be single-run evaluations. The '± SD' values in the paper are computed across model sizes or across models, not across experimental runs. There is no way to assess the stability of individual model evaluations."
     },
     {
       "flag": "Misleading 'seeds' terminology",
       "detail": "Section 5.1.1 refers to 'mean ± SD over 6 seeds' when computing statistics across the 6 model sizes (0.5B through 32B) in a model series. This language is misleading as 'seeds' typically refers to random seeds in ML research."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "All claims about trends (scaling reduces MRI, SFT increases memorization, PPO is better than SFT) are based on comparing point estimates without any formal statistical testing."
     },
     {
       "flag": "GPT-5 dependency for task generation",
       "detail": "All three perturbation types (code rewriting, mutation, paraphrase) and the quality judge are generated by GPT-5. Any systematic biases in GPT-5's generation could affect the evaluation. The paper does not analyze sensitivity to the generating model."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section despite several potential threats: narrow model families tested (only Qwen and Llama), Python-only evaluation, reliance on GPT-5 for data generation, and small test split for MBPP+ (n=78)."
     },
     {
       "flag": "Limited model diversity for broad claims",
       "detail": "The scaling analysis covers only Qwen-2.5 and Llama families. Claims about memorization trends with model scale are presented broadly but tested on a limited set of architectures."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval and Codex, foundational benchmark and model for LLM code generation evaluation."
     },
     {
       "title": "Is your code generated by chatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Introduces EvalPlus/MBPP+, one of the two primary benchmarks used in this paper for evaluating code generation."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "relevance": "Introduces BIGCODEBENCH, the harder benchmark used to study memorization persistence on complex tasks."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free benchmark using temporally fresh problems, directly relevant to the memorization vs generalization debate."
     },
     {
       "title": "Unveiling memorization in code models",
       "authors": ["Zhou Yang", "Zhipeng Zhao", "Chenyu Wang"],
       "year": 2024,
       "doi": "10.1145/3597503.3639074",
       "relevance": "Studies general memorization in code models, finding it increases with model size — a claim this paper challenges for harmful memorization."
     },
     {
       "title": "Traces of memorisation in large language models for code",
       "authors": ["Ali Al-Kaswan", "Maliheh Izadi", "Arie van Deursen"],
       "year": 2024,
       "doi": "10.1145/3597503.3639133",
       "relevance": "Measures general memorization in code LLMs via regurgitation, finding it increases with scale — contrasted with this paper's findings on harmful memorization."
     },
     {
       "title": "Quantifying contamination in evaluating code generation capabilities of language models",
       "authors": ["Martin Riddell", "Ansong Ni", "Arman Cohan"],
       "year": 2024,
       "arxiv_id": "2403.04811",
       "relevance": "Quantifies benchmark contamination in code generation evaluation, directly relevant to the contamination problem this paper addresses."
     },
     {
       "title": "ReCode: Robustness evaluation of code generation models",
       "authors": ["Shiqi Wang", "Li Zheng", "Haifeng Qian"],
       "year": 2022,
       "arxiv_id": "2212.10264",
       "relevance": "Proposes robustness evaluation through semantic-preserving perturbations — the mutation and paraphrase baselines in this paper are adapted from ReCode."
     },
     {
       "title": "The pitfalls of memorization: When memorization hurts generalization",
       "authors": ["Reza Bayat", "Mohammad Pezeshki", "Elvis Dohmatob"],
       "year": 2024,
       "arxiv_id": "2412.07684",
       "relevance": "Theoretical framework for when memorization hurts generalization, directly informing this paper's definition of harmful memorization."
     },
     {
       "title": "Generalization or memorization: Data contamination and trustworthy evaluation for large language models",
       "authors": ["Yihong Dong", "Xue Jiang", "Huanyu Liu"],
       "year": 2024,
       "arxiv_id": "2402.15938",
       "relevance": "Studies data contamination and its effect on trustworthy LLM evaluation, measuring memorization via surface/structural overlap."
     },
     {
       "title": "On memorization of large language models in logical reasoning",
       "authors": ["Chulin Xie", "Yangsibo Huang", "Chiyuan Zhang"],
       "year": 2024,
       "arxiv_id": "2410.23123",
       "relevance": "Studies LLM memorization in reasoning tasks, extending the memorization vs generalization question beyond code generation."
     },
     {
       "title": "Qwen2.5-coder technical report",
       "authors": ["Binyuan Hui", "Jian Yang", "Zeyu Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Technical report for the Qwen-2.5-Coder model family, one of the primary model series evaluated in this paper."
     },
     {
       "title": "Measuring memorization in RLHF for code completion",
       "authors": ["Aneesh Pappu", "Billy Porter", "Ilia Shumailov", "Jamie Hayes"],
       "year": 2024,
       "arxiv_id": "2406.11715",
       "relevance": "Studies memorization specifically in RLHF-trained code models, relevant to this paper's comparison of SFT vs PPO effects on memorization."
     }
   ]
 }