ai-research-survey

scan.json (29045B)

---

 {
   "paper": {
     "title": "LessLeak-Bench: A First Investigation of Data Leakage in LLMs Across 83 Software Engineering Benchmarks",
     "authors": [
       "Xin Zhou",
       "Martin Weyssow",
       "Ratnadira Widyasari",
       "Ting Zhang",
       "Junda He",
       "Yunbo Lyu",
       "Jianming Chang",
       "Beiqi Zhang",
       "Dan Huang",
       "David Lo"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2502.06215",
     "doi": "10.48550/arXiv.2502.06215"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "Data leakage in 83 SE benchmarks is generally minimal (avg 4.8% Python, 2.8% Java, 0.7% C/C++), but some benchmarks are severely affected—QuixBugs at 100% and BigCloneBench at 55.7%. Leakage significantly inflates evaluation metrics: StarCoder-7b achieves Pass@1 4.9x higher on leaked vs non-leaked APPS samples. Four causes of high leakage are identified: direct inclusion of benchmark repos in pre-training data, repository overlap, reliance on coding platforms like LeetCode, and shared data sources like GitHub issues. Perplexity-based automated leakage detection is ineffective, achieving only 40-50% accuracy.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper introduces DetectLeak and LessLeak-Bench but provides no repository URL or download link for their code or cleaned benchmarks in the paper text."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "LessLeak-Bench and AutoDetectLeak-Bench are described but no download URL or data archive is provided in the paper. The paper references StarCoder's pre-training data on HuggingFace but does not release their own labeled data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'NVIDIA GeForce A5000 GPU with 24 GB of memory' and references the BigCode Team's implementation, but provides no requirements.txt, dependency list, or sufficient detail to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology is described at a conceptual level but without runnable scripts, commands, or a README."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All leakage ratios (Tables 2-4) and Pass@k results (Table 7) are reported as point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'StarCoder-7b achieves a Pass@1 score that is 4.9 times higher on leaked samples' but provides no statistical significance tests for any comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with context: '4.9 times higher' with actual values (4.4% vs 0.9% Pass@1), '5.6 and 6.0 times higher' for Pass@2 and Pass@3 (Table 7, Section 5.3). Leakage ratios are reported as proportions with counts."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the number of benchmarks studied (83), number of annotators (8), or the size of AutoDetectLeak-Bench (1,300 samples). No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experimental results. Pass@k results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 discusses and compares against alternative approaches (Exact Match, code clone detection tools) and justifies choosing MinHash+LSH. RQ3 compares leaked vs non-leaked subsets. RQ4 evaluates Perplexity as a detection method. Related work (Section 7) compares against Yang et al. and López et al."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Related work includes 2024 publications (López et al., Riddell et al., Matton et al.). MinHash+LSH is the method used by contemporary LLMs (StarCoder, StarCoder2, Qwen) for deduplication."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The DetectLeak pipeline has multiple components (automated detection + manual verification) but no ablation study removes individual components to measure their contribution. Figure 3 shows overall filtering statistics but not a controlled ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: leakage count, leakage ratio (Section 3.3), Pass@1/2/3 (Table 7), Cohen's Kappa for annotator agreement (0.9424), and top-k accuracy for Perplexity detection (Section 5.4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Eight experienced annotators (3 postdocs, 4 PhD, 1 Master's) manually labeled 6,643 potential duplicate pairs flagged by MinHash+LSH, with two independent annotators per pair and a third resolving conflicts (Section 3.2)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For RQ3, the APPS test set (5,000 samples, separate from 5,000 training samples) is used. For RQ4, AutoDetectLeak-Bench is constructed from manually labeled data as a separate evaluation set (Section 5.4)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by programming language (Tables 2-4), by individual benchmark (83 benchmarks), by SE task type (Section 5.1.4), and by model size (StarCoder-7b/3b/1b in Table 7)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 reports that Perplexity-based detection achieves only 40-50% accuracy, a clear failure case. Figure 3 shows 72% of auto-flagged pairs are non-duplicates (false positive analysis). Section 6.3 acknowledges the automated tool may miss cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 is essentially a negative result: Perplexity-based automated leakage detection is unreliable (40-50% accuracy). The paper also reports that most SE benchmarks have minimal leakage, which is a 'negative' finding relative to the concern being investigated."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: average leakage ratios of 4.8%/2.8%/0.7% match Tables 2-4; QuixBugs 100% and BigCloneBench 55.7% match Table 2/3; StarCoder-7b 4.9x higher Pass@1 on leaked samples matches Table 7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'data leakage has a significant impact on the evaluation of LLMs' (Section 5.3), which is a causal claim. However, the leaked vs non-leaked comparison is observational—leaked APPS samples may differ systematically from non-leaked in difficulty, domain, or complexity. This confound is not addressed."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Data Leakage in LLMs' broadly but only StarCoder is studied. Section 6.3 acknowledges limitations ('may not generalize to all SE benchmarks and LLMs'), but the paper also claims generalizability to derivative LLMs (WizardCoder, OctoPack, CodeShell, DeepSeek-Coder) based on shared pre-training data, which is speculative."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "For RQ3 (leaked vs non-leaked performance gap), the paper does not discuss alternative explanations such as systematic difficulty differences between leaked and non-leaked samples. The threats to validity section (6.3) discusses representativeness but not alternative explanations for observed effects."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper defines data leakage as inclusion of evaluation data in pre-training data (Section 2.1), which is exactly what MinHash+LSH detects. Their measurement (near-duplicate overlap with pre-training corpus) directly maps to their definition. RQ3 separately tests whether this overlap actually affects performance, demonstrating awareness of the gap between presence in training data and behavioral impact."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "StarCoder-7b, StarCoder-3b, and StarCoder-1b are specified with sizes. StarCoder is referenced via its paper [45] and HuggingFace homepage. These are specific, versioned models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 provides the exact prompt format: '### Instruction: [instruction input from APPS data sample] ### Response:'. The fill values come from the public APPS dataset, making prompts fully reconstructable."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "MinHash+LSH hyperparameters are reported (n-gram=2, Jaccard threshold=0.7). However, for the StarCoder evaluation in RQ3, no LLM inference parameters are stated (temperature, top-p, max tokens, sampling strategy)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. StarCoder models are prompted directly in zero-shot mode."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is well-documented: benchmark selection from a survey of 395 papers (Section 4.2), MinHash+LSH detection with 1.7 trillion comparisons, manual labeling with 8 annotators, 6,643 flagged pairs → 1,950 real duplicates and 4,741 non-duplicates (Figure 2, Section 3). For RQ4, deduplication and under-sampling are described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 'Threats to Validity' provides substantive discussion of limitations including generalizability, automated technique limitations, and LLM selection rationale."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 identifies specific threats: 'limited to the specific SE benchmarks and LLMs studied,' the automated technique 'may not detect all leaked SE benchmark samples,' and explains the specific rationale for StarCoder selection."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states scope boundaries: focus on pre-training leakage (not fine-tuning, Section 2.1), three programming languages only, StarCoder family only, and 83 specific benchmarks. Section 6.3 explicitly states results 'may not generalize to all SE benchmarks and LLMs.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The manually labeled duplicate pairs, AutoDetectLeak-Bench, and LessLeak-Bench are described but no download URL or data archive is provided in the paper text for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail: benchmarks selected from a survey of 395 papers (Section 4.2), StarCoder pre-training data from The Stack on HuggingFace (Section 4.1), manual annotation process with 8 annotators (Section 3.2)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The 8 annotators are characterized: 'three post-doctoral researchers, four PhD students, and one Master's student, each with a minimum of four years of experience in programming' (Section 3.2)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 illustrates the full DetectLeak pipeline. Each step includes counts: 1.7 trillion comparisons → 6,643 auto-flagged pairs → 1,950 verified duplicates → 606/816/108 leaked samples for Python/Java/C++. Cohen's Kappa reported (0.9424)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgement section states: 'supported by the National Research Foundation, under its Investigatorship Grant (NRF-NRFI08-2022-0002).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are disclosed: primarily Singapore Management University, with co-authors at Southeast University (China) and Wuhan University (China). Authors are not affiliated with any LLM company being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The National Research Foundation of Singapore is a government research funding body with no financial stake in whether SE benchmarks exhibit data leakage."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states StarCoder's pre-training data is from The Stack, 'collected from public GitHub repositories between 2015 and 2022.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Train/test overlap is the central topic of the entire paper. The DetectLeak framework systematically identifies overlap between pre-training data and 83 SE benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark contamination is the core research question. The paper quantifies exactly how much of each benchmark appeared in pre-training data and introduces LessLeak-Bench to remove contaminated samples."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The 8 annotators are research team members performing data labeling, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyzes code data and LLM pre-training datasets."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Annotator qualifications are described but they are researchers, not study participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; not an experimental study with human subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; not an experimental study with human subjects."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '1.7 trillion comparisons' but does not report actual compute time, cost, or wall-clock time for any stage of the pipeline or the LLM inference experiments."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The GPU type is mentioned (NVIDIA GeForce A5000, 24GB) but total compute time, GPU hours, or overall budget for the extensive experiments is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The Pass@k experiments in RQ3 (Table 7) do not report results across multiple random seeds. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs for the LLM evaluation in RQ3 is not stated. It is unclear whether results are from one or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "MinHash+LSH parameters (n-gram=2, threshold=0.7) were set 'based on initial small-scale trials' (Section 3.1) but the number of configurations tried and selection criteria are not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The MinHash+LSH configuration is justified only as 'based on initial small-scale trials' without describing the trials, how many configurations were tested, or the selection criterion."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own DetectLeak framework without acknowledging self-evaluation bias. No independent evaluation or discussion of author-evaluation bias is present."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Three model sizes are compared (StarCoder-7b/3b/1b) in Table 7 but performance is not discussed as a function of compute budget. The compute cost differences across model sizes are not addressed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses APPS for RQ3 evaluation without discussing whether Pass@k on APPS adequately measures the impact of data leakage, or whether the leaked/non-leaked split introduces systematic differences beyond leakage (e.g., difficulty confound)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. StarCoder models are evaluated directly with zero-shot prompting."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses temporal leakage: StarCoder's pre-training data is from 2015-2022 (Section 4.1), and they identify that many benchmarks were created before this period, enabling leakage. Table 6 traces causes to benchmark creation dates and pre-training data overlap."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper focuses on data leakage (presence of benchmark data in pre-training) but does not discuss feature leakage—whether evaluation inputs contain information not available at prediction time in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Non-independence between pre-training and benchmark data is the central research question. The paper systematically identifies and quantifies overlap between StarCoder's pre-training corpus and 83 SE benchmarks."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper develops DetectLeak, a concrete leakage detection pipeline combining MinHash+LSH near-duplicate detection with manual verification by 8 annotators (Section 3, Figure 2). They also evaluate Perplexity-based detection as an alternative (RQ4)."
       }
     }
   },
   "claims": [
     {
       "claim": "Data leakage in SE benchmarks is generally minimal, with average leakage ratios of only 4.8% (Python), 2.8% (Java), and 0.7% (C/C++).",
       "evidence": "Tables 2-4 present per-benchmark leakage ratios computed from 1,950 manually verified duplicate pairs out of 6,643 auto-flagged pairs across 83 benchmarks (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "Some benchmarks exhibit very high leakage: QuixBugs has 100% leakage, BigCloneBench has 55.7%.",
       "evidence": "Table 2 shows QuixBugs (40/40 samples leaked), Table 3 shows BigCloneBench (508/912 leaked). Root causes identified in Table 6 (Section 5.2).",
       "supported": "strong"
     },
     {
       "claim": "Data leakage significantly inflates LLM evaluation metrics: StarCoder-7b achieves Pass@1 4.9 times higher on leaked APPS samples than non-leaked.",
       "evidence": "Table 7 shows StarCoder-7b Pass@1 of 4.4% on leaked vs 0.9% on non-leaked APPS test samples (Section 5.3). Similar ratios for StarCoder-3b and 1b.",
       "supported": "moderate"
     },
     {
       "claim": "Four primary causes of high data leakage: direct inclusion of benchmark repos, repository overlap, coding platform dependence (LeetCode), and shared data sources (GitHub issues).",
       "evidence": "Section 5.2, Tables 5-6 analyze the top leaked benchmarks and trace leakage to specific repositories and patterns (e.g., PatrickShaw/QuixBugs, LeetCode-named repos).",
       "supported": "moderate"
     },
     {
       "claim": "Perplexity-based automated leakage detection without access to pre-training data is ineffective, with accuracy only 40-50%.",
       "evidence": "Figure 5 shows detection accuracy across top-k rankings for StarCoder-7b/3b/1b, all achieving near-chance accuracy (Section 5.4). Figure 4 shows no clear separation in Perplexity distributions.",
       "supported": "moderate"
     },
     {
       "claim": "Program repair benchmarks are particularly vulnerable to data leakage, with an average leakage rate of 12.5% versus 0.62% for code generation.",
       "evidence": "Section 5.1.4 reports that only 1/9 program repair benchmarks has 0% leakage, while 26/33 code generation benchmarks have 0% leakage.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Confound in leaked vs non-leaked comparison",
       "detail": "RQ3 compares LLM performance on leaked vs non-leaked APPS samples, but leaked samples may systematically differ from non-leaked in difficulty, domain, or problem type. This confound is not discussed—the performance gap could partially reflect difficulty differences, not just memorization from leakage."
     },
     {
       "flag": "Single LLM family studied despite broad title",
       "detail": "The title says 'Data Leakage in LLMs' but only StarCoder is studied. Claims of generalizability to derivative models (WizardCoder, DeepSeek-Coder) are speculative since derivative models may deduplicate differently or use additional data."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "The 4.9x performance gap on leaked vs non-leaked data (Table 7) and all comparative claims lack statistical significance testing. With small leaked sample sizes for some benchmarks, observed differences could be unstable."
     },
     {
       "flag": "Missing artifact release",
       "detail": "LessLeak-Bench, DetectLeak, and AutoDetectLeak-Bench are introduced as contributions but no repository URL, data download link, or archive is provided in the paper text."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark, one of the most widely used code generation benchmarks studied for leakage in this paper."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig"],
       "year": 2024,
       "relevance": "Major SE benchmark for issue fixing that the paper finds has 8.7% leakage ratio, raising concerns for LLM evaluation."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["R. Li", "L. B. Allal", "Y. Zi"],
       "year": 2023,
       "relevance": "The target LLM for this study, chosen for its fully open pre-training data enabling concrete leakage analysis."
     },
     {
       "title": "On inter-dataset code duplication and data leakage in large language models",
       "authors": ["J. A. H. López", "B. Chen", "M. Saaz"],
       "year": 2024,
       "relevance": "Prior work on data leakage in smaller LLMs (CodeBERT), limited to 3 benchmarks—this paper extends to 83 benchmarks and larger models."
     },
     {
       "title": "Unveiling memorization in code models",
       "authors": ["Z. Yang", "Z. Zhao", "C. Wang"],
       "year": 2024,
       "relevance": "Studies memorization in CodeParrot using clone detection, related but distinct focus from benchmark-specific data leakage."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["J. Austin", "A. Odena", "M. I. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark, widely used for code generation evaluation and studied for leakage."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["N. Jain", "K. Han", "A. Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Benchmark designed to be contamination-free using temporal splits, found to have 0% leakage in this study."
     },
     {
       "title": "Leak, cheat, repeat: Data contamination and evaluation malpractices in closed-source LLMs",
       "authors": ["S. Balloccu", "P. Schmidtová", "M. Lango"],
       "year": 2024,
       "arxiv_id": "2402.03927",
       "relevance": "Addresses data contamination in closed-source LLMs, complementary to this paper's focus on open-source LLMs."
     },
     {
       "title": "Quantifying contamination in evaluating code generation capabilities of language models",
       "authors": ["M. Riddell", "A. Ni", "A. Cohan"],
       "year": 2024,
       "relevance": "Studies leakage in HumanEval and MBPP using Levenshtein similarity, a gray-literature baseline for this work."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["D. Hendrycks", "S. Basart", "S. Kadavath"],
       "year": 2021,
       "relevance": "Introduces APPS benchmark used in RQ3 to demonstrate the impact of data leakage on LLM evaluation metrics."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "authors": ["D. Guo"],
       "year": 2024,
       "relevance": "Derivative LLM using data curation techniques inspired by StarCoder, illustrating how leakage findings propagate to other models."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["X. Hou", "Y. Zhao", "Y. Liu"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLMs for SE tasks that served as the source for benchmark selection in this study."
     }
   ]
 }