ai-research-survey

scan-v5.json (26640B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "LessLeak-Bench: A First Investigation of Data Leakage in LLMs Across 83 Software Engineering Benchmarks",
     "authors": [
       "Xin Zhou",
       "Martin Weyssow",
       "Ratnadira Widyasari",
       "Ting Zhang",
       "Junda He",
       "Yunbo Lyu",
       "Jianming Chang",
       "Beiqi Zhang",
       "Dan Huang",
       "David Lo"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2502.06215",
     "doi": "10.48550/arXiv.2502.06215"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All major abstract claims — average leakage ratios (4.8%/2.8%/0.7%), specific high-leakage benchmarks (QuixBugs 100%, BigCloneBench 55.7%), and the 4.9× inflation finding — are directly supported by Tables 2–4 and Table 7.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that leakage inflates metrics is tested by splitting APPS into leaked vs. non-leaked subsets and comparing Pass@k on the same models; this controlled comparison is appropriate for the claim, though confounds like problem difficulty are not ruled out.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states findings are limited to StarCoder (one fully open-source LLM family) and 83 SE benchmarks; the threats-to-validity section acknowledges results may not generalize to all LLMs and benchmarks.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether leaked samples might be systematically easier problems (a plausible confound), attributing all performance inflation solely to memorization without ruling out difficulty bias.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Pass@k measures functional correctness directly via test case execution; leakage count/ratio measures data overlap directly. The paper does not conflate proxy measures with downstream claims.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 'Threats to Validity' is a dedicated subsection discussing generalizability and detection completeness limitations.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats named include: restriction to one LLM (StarCoder) and 83 benchmarks, and that MinHash+LSH may miss leakage cases since it operates on near-duplicates rather than semantic equivalents.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds results to StarCoder's pre-training data (The Stack, 2015–2022 GitHub) and three programming languages across 83 benchmarks, stating findings on proprietary LLMs cannot be verified.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgment section discloses funding from Singapore's National Research Foundation under Investigatorship Grant NRF-NRFI08-2022-0002.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed (Singapore Management University, Southeast University, Wuhan University) and none have affiliation with the evaluated LLM providers.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NRF Singapore is a government research funder with no commercial stake in LLM benchmark validity findings.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial interests declaration beyond the funding acknowledgment; no explicit 'no competing interests' statement is present.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 formally defines 'data leakage' with mathematical notation distinguishing exact leakage and semantic leakage, and Section 2.2 distinguishes data leakage detection from code clone detection.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The contribution is explicitly stated: first large-scale analysis of leakage across 83 SE benchmarks, DetectLeak framework, AutoDetectLeak-Bench labeled dataset, and LessLeak-Bench cleaned benchmarks.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 specifically contrasts this work with Yang et al., Lopez et al., Matton et al., and Riddell et al. on scale (83 vs. 2–3 benchmarks), model size (StarCoder vs. lightweight LLMs), and methodology (manual verification vs. automated tools only).",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "DetectLeak framework is described methodologically but no GitHub URL or code release link is provided anywhere in the paper.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "LessLeak-Bench and AutoDetectLeak-Bench are introduced but no download URL or repository link is provided; the paper only describes their creation.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Only a GPU model (NVIDIA A5000, 24GB) and two library names (BigCode DataSketches) are mentioned; no requirements.txt, Dockerfile, or complete dependency list is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "The three-phase DetectLeak pipeline (automated detection → manual labeling → quantification) is described conceptually but no step-by-step instructions or runnable scripts are provided.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Leakage ratios in Tables 2–4 and Pass@k scores in Table 7 are presented as point estimates with no confidence intervals or error bars.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "Table 7 compares Pass@k on leaked vs. non-leaked subsets across three model sizes with no statistical significance tests; the 4.9× difference is reported without testing.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Effect sizes are reported as ratio comparisons (e.g., '4.9 times higher Pass@1', '5.6 times higher Pass@2') rather than just raw differences, providing context for the magnitude.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The 83 benchmark selection is justified by coverage of a prior survey, but there is no power analysis and the APPS-only analysis for RQ3 (108 leaked samples) has no sample size justification.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "All results are point estimates; no standard deviations, variance, or spread metrics are reported across runs or annotation cycles.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Non-leaked sample performance serves as the natural baseline for RQ3; random baseline (50%) serves as comparison for RQ4 perplexity detection accuracy.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "StarCoder-7b/3b/1b are the same models whose pre-training data is under investigation, making them the appropriate and contemporary models for the comparison.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": false,
           "answer": false,
           "justification": "This is an empirical investigation with a pipeline, not a system with components to ablate; the MinHash threshold sensitivity is partially explored but not as a formal ablation.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "The paper uses leakage count, leakage ratio, Pass@1/2/3, Perplexity, Cohen's Kappa, and Jaccard similarity across different research questions.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Manual labeling by 8 experienced developers (postdocs, PhD/Master's students) with inter-rater agreement measured via Cohen's Kappa (0.9424) is the core validation mechanism.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "RQ3 experiments use the APPS test set (5,000 samples) split into leaked and non-leaked subsets, which is the held-out test portion of the benchmark.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by programming language (Tables 2–4), by SE task category (Section 5.1.4), and individually per benchmark with full leakage statistics.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The paper discusses why models don't achieve perfect performance on leaked samples (problem complexity limits memorization) and why perplexity fails to distinguish leaked from non-leaked (Section 5.4).",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "RQ4 is a negative result: perplexity-based automated detection achieves only 40–50% accuracy across all model sizes, barely above random, explicitly framed as a challenge for future work.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "StarCoder (15.5B), StarCoder-7b, StarCoder-3b, StarCoder-1b are specified by name and size; the pre-training data source (The Stack) is identified with its HuggingFace URL.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "The exact prompt format used for APPS experiments is given: '### Instruction: [instruction input] ### Response:' with zero-shot specification.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "MinHash parameters (n-gram=2, Jaccard threshold=0.7) are reported, but LLM inference hyperparameters (temperature, top-p, number of samples for Pass@k) are not disclosed.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used; the study evaluates LLMs in a straightforward zero-shot prompting setup.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 3 and Figure 2 document the full three-phase pipeline including automated detection, manual annotation process, and conflict resolution procedure with details on annotation categories.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "AutoDetectLeak-Bench is described as a resource to be shared but no repository link, DOI, or download URL is provided in the paper.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.2 describes the benchmark selection process (leveraging a survey of 395 papers from 2017–2024, expanding via citations, requiring replication packages), with explicit inclusion/exclusion criteria.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "The benchmarks are standard public datasets, not recruited participant data; annotators are described by role but this is not a participant recruitment study.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Figure 2 and Sections 3.1–3.3 document the full pipeline from raw pre-training data comparison through automated detection, manual annotation, conflict resolution, to leakage quantification.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": true,
           "justification": "StarCoder's pre-training data (The Stack) is explicitly described as GitHub repositories collected between 2015 and 2022, establishing a clear data collection period.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "This is the entire focus of the paper — systematically identifying and quantifying overlap between pre-training data and benchmark test sets across 83 benchmarks.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Contamination is directly measured through MinHash+LSH comparison of 1.7 trillion pairs, manually verified, and quantified per benchmark with specific counts and ratios.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants as research subjects; annotators are part of the research team, not subjects.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects research; IRB not applicable.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants as subjects.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants as subjects.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants as subjects.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants as subjects.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants as subjects.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference cost, latency, or GPU hours are reported for running StarCoder models on the APPS benchmark experiments.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Only the GPU hardware (NVIDIA A5000, 24GB) is mentioned; total compute hours or cost for processing 1.7 trillion pairs is not stated.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Data leakage in SE benchmarks is generally minimal, with average leakage ratios of only 4.8%, 2.8%, and 0.7% for Python, Java, and C/C++ benchmarks respectively.",
       "evidence": "Tables 2–4 show per-benchmark leakage counts and ratios across 83 benchmarks, with averages computed at the bottom of each table.",
       "supported": "strong"
     },
     {
       "claim": "Some benchmarks exhibit very high leakage: QuixBugs has 100% leakage and BigCloneBench has 55.7% leakage.",
       "evidence": "Table 2 shows QuixBugs with 40/40 samples leaked (100%), Table 3 shows BigCloneBench with 508/912 samples leaked (55.7%), both verified through manual labeling.",
       "supported": "strong"
     },
     {
       "claim": "Data leakage substantially inflates LLM evaluation metrics: StarCoder-7b achieves Pass@1 that is 4.9 times higher on leaked samples than non-leaked samples on APPS.",
       "evidence": "Table 7 shows StarCoder-7b Pass@1 of 4.4% on leaked vs. 0.9% on non-leaked; pattern holds across StarCoder-3b and StarCoder-1b and across Pass@1/2/3.",
       "supported": "strong"
     },
     {
       "claim": "Program repair benchmarks have significantly higher average leakage (12.5%) compared to code generation benchmarks (0.62%).",
       "evidence": "Section 5.1.4 reports task-level analysis: only 1/9 program repair benchmarks has 0% leakage vs. 26/33 code generation benchmarks with 0% leakage.",
       "supported": "strong"
     },
     {
       "claim": "Perplexity-based automated leakage detection is unreliable, achieving only 40–50% accuracy across model sizes.",
       "evidence": "Figure 5 shows top-k detection accuracy ranging 40–50% for StarCoder-7b/3b/1b across k=100–1000, barely above the 50% random baseline.",
       "supported": "strong"
     },
     {
       "claim": "High leakage is primarily caused by four factors: direct benchmark inclusion, source repository overlap, LeetCode derivation, and shared GitHub issue data sources.",
       "evidence": "Table 6 traces each high-leakage benchmark to a specific cause; Table 5 shows 'leetcode' appears in repository names 8 times in top 20 leakage-contributing repos.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "The first large-scale study of data leakage across 83 SE benchmarks reveals that average leakage is low (4.8%/2.8%/0.7% for Python/Java/C++) but highly uneven, with QuixBugs fully leaked (100%) and BigCloneBench 55.7% leaked into StarCoder's pre-training data. Leakage has a large practical impact: StarCoder-7b achieves 4.9× higher Pass@1 on leaked APPS samples, confirming that contamination inflates reported metrics. Four root causes are identified (direct inclusion, repository overlap, LeetCode derivation, shared GitHub issues), and the cleaned LessLeak-Bench is introduced as a mitigation. Automated detection via perplexity scores performs near chance (40–50%), leaving accurate leakage detection without pre-training data access as an open problem.",
   "red_flags": [
     {
       "flag": "Single LLM generalizability",
       "detail": "All empirical leakage measurements are restricted to StarCoder and The Stack pre-training data; results may not transfer to GPT-4, Llama3, or other models with different (undisclosed) pre-training corpora."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Table 7's key comparison (leaked vs. non-leaked Pass@k) reports large effect sizes without any statistical tests, making it impossible to assess whether differences on small leaked sample counts are robust."
     },
     {
       "flag": "Difficulty confound not ruled out",
       "detail": "Leaked samples may be systematically easier problems (e.g., classic LeetCode-style algorithms) rather than harder novel tasks; no difficulty-controlled analysis is presented to isolate memorization from task difficulty."
     },
     {
       "flag": "RQ3 analysis on single benchmark",
       "detail": "The claim that leakage inflates metrics is demonstrated only on APPS; other high-leakage benchmarks (QuixBugs, BigCloneBench) are not used to validate the effect size claim."
     },
     {
       "flag": "No code or data release URL",
       "detail": "DetectLeak framework, LessLeak-Bench, and AutoDetectLeak-Bench are described but no repository links are provided, undermining reproducibility claims."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "High-leakage benchmark (8.7%) studied in the paper; central example of issue-fix benchmark with pre-training overlap."
     },
     {
       "title": "StarCoder: May the Source Be With You!",
       "relevance": "The target LLM whose pre-training data (The Stack) is the reference dataset for all leakage measurements."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval/Codex)",
       "relevance": "HumanEval is one of the studied benchmarks (1.8% leakage) and a widely used code generation benchmark."
     },
     {
       "title": "On Inter-Dataset Code Duplication and Data Leakage in Large Language Models",
       "relevance": "Direct prior work on data leakage in lightweight LLMs (CodeBERT); this paper extends to larger models and 83 benchmarks."
     },
     {
       "title": "Quantifying Contamination in Evaluating Code Generation Capabilities of Language Models",
       "relevance": "Prior gray-literature work on HumanEval/MBPP leakage; this paper extends scope and adds manual verification."
     },
     {
       "title": "Measuring Coding Challenge Competence with APPS",
       "relevance": "APPS is the primary benchmark used for RQ3 impact analysis (10.8% leakage, 4.9× performance inflation)."
     },
     {
       "title": "Program Synthesis with Large Language Models (MBPP)",
       "relevance": "MBPP is one of the studied benchmarks (0.4% leakage) and a standard code generation evaluation benchmark."
     },
     {
       "title": "The Stack: 3 TB of Permissively Licensed Source Code",
       "relevance": "StarCoder's pre-training dataset; the reference corpus against which all 83 SE benchmarks are compared for leakage."
     },
     {
       "title": "QuixBugs: A Multi-Lingual Program Repair Benchmark Set",
       "relevance": "The highest-leakage benchmark studied (100%), used as a central example of complete benchmark contamination."
     },
     {
       "title": "Towards a Big Data Curated Benchmark of Inter-Project Code Clones (BigCloneBench)",
       "relevance": "Second highest-leakage benchmark (55.7%), used as a key example of repository-overlap-driven contamination."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Any researcher evaluating LLMs on SE benchmarks should know which benchmarks are contaminated; LessLeak-Bench is a drop-in replacement they can use immediately."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that overall leakage is low (surprising to pessimists) but specific widely-used benchmarks like QuixBugs are 100% contaminated (alarming to practitioners) challenges simple narratives."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Benchmark contamination undermines AI evaluation validity but is not a direct safety risk; the concern is methodological integrity rather than harm."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "The finding that SWE-bench (a benchmark widely used in high-profile agent competitions) has 8.7–10.6% leakage creates some controversy for the community relying on it."
     },
     "demo_ability": {
       "score": 1,
       "justification": "LessLeak-Bench is described as available but no URL is provided; practitioners cannot easily try the cleaned benchmarks without contacting the authors."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Singapore Management University is a respected SE research group (David Lo's lab) but not a globally prominent AI lab name."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "46908281",
         "title": "LLMs do plan before they genenrate tokens",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46908281",
         "created_at": "2026-02-06T02:30:48Z"
       },
       {
         "hn_id": "43025980",
         "title": "Emergent Response Planning in LLM",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43025980",
         "created_at": "2025-02-12T14:57:10Z"
       }
     ],
     "top_points": 3,
     "total_points": 4,
     "total_comments": 0
   }
 }