ai-research-survey

scan.json (20466B)

---

 {
   "paper": {
     "title": "Concerned with Data Contamination? Assessing Countermeasures in Code Language Model",
     "authors": ["Jialun Cao", "Wuqi Zhang", "Shing-Chi Cheung"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2403.16898"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive link is provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes collecting 2.49M Python functions but does not provide a download link or archive for the dataset."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'two NVIDIA RTX 6000 Ada GPUs, each with 48GB' but does not provide library versions, requirements.txt, or environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, scripts, or README are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables II and III report only point estimates (Pass@k scores) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares performance across code groups and claims differences but does not use any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table III reports absolute differences (delta columns) between original and refactored code groups, with baseline context (e.g., '38.5 → 35.1, Δ=-3.4')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C states '95% confidence level and 5% margin of error' resulting in 384 sampled functions per year, explicitly justifying the sample size."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. Results appear to be from nucleus sampling but no run-level variance is shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study compares CLM performance across contaminated vs. cleansed data groups, serving as the baseline comparison design central to the paper."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Eight CLMs are evaluated including contemporary models like GPT-3.5, GitHub Copilot, Phind-CodeLlama-34b, and WizardCoder (Table I)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 systematically tests five individual refactoring operators (IFF, Loop, Renm, Param, Deco) applied independently to isolate each operator's effect (Section II-A3, Table III)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Pass@1, Pass@3, Pass@5 are used for performance evaluation. Six MIA-related metrics (perplexity, ppl_lower, zlib entropy, MIN-K% at 5/10/20) are used in RQ4. Code complexity uses cyclomatic and cognitive complexity."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated using exact string matching for code completion correctness. No human evaluation of outputs is conducted."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The temporal splitting design ensures cleansed data (post-cutoff) was not in training data. The study's core design is about separating contaminated from clean test data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model, per year (Code-2018 through Code-2023), per curated dataset (HumanEval, CoderEval), and per refactoring operator (Tables II and III)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses counter-intuitive findings where countermeasures fail (e.g., CLMs performing better on recent/curated data, loop transformation increasing performance). These are essentially failure cases of the countermeasures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The main finding is negative: existing countermeasures (recent data, curated data, refactoring) do not reliably mitigate data contamination. MIA metrics also shown to be ineffective (Finding 8)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (CLMs sometimes perform better on post-cutoff data, refactoring can improve performance, MIA metrics cannot distinguish contaminated/cleansed data) are all supported by Tables II, III, and Figure 6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal suggestions (e.g., 'the popularity of AI programming assistants such as Copilot may further exacerbate data contamination') based on observational evidence without controlling for confounds. The Copilot explanation for Code-2023 performance is speculative."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The threats to validity section explicitly bounds results to Python and code completion tasks, acknowledging these may not generalize to other languages or tasks (Section V)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-E discusses alternative explanations for Code-2023 performance: copy-paste culture vs. AI coding assistants. The paper also considers code complexity differences as a confound and measures them (Section IV-A)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table I specifies model sizes and release dates. Section III-A specifies 'ChatGPT3.5-turbo-0613' with specific version. Model commit histories are traced for release dates."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The task is code completion (prefix → suffix prediction), not prompt-based. The models receive code prefixes directly without crafted prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV states 'nucleus sampling... five solution samples are randomly generated with a temperature of 0.2.' MIN-K% parameters (K=5.0, 10.0, 20.0) are also specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated directly on code completion."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-B3 describes filtering: extracting functions from Python code, filtering out functions inside classes/other functions, filtering empty functions. Section III-B details data collection from Stack v2 and GitHub crawling with specific criteria (50+ stars, permissive licenses)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V 'Threats to Validity' provides a dedicated discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section V discusses specific threats: Python language selection bias, code completion task limitation (masking only last statement), and possible semantic overlap between code groups with their mitigation (similarity metrics)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope is limited to Python, code completion tasks, and the specific set of CLMs tested. Section V calls for 'further research to explore a wider array of coding tasks.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 2.49M Python functions dataset and sampled 2,304 functions are not made available for download."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-B describes data collection in detail: Stack v2 for pre-March 2022 data, GitHub crawling for post-March 2022 data (repos with 50+ stars, permissive licenses, created April 2022 - December 2023). Time range and total counts provided."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is collected from public code repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: Stack v2 + GitHub crawl → extract Python functions → filter (remove nested/empty functions) → sample 384 per year at 95% confidence → apply to CLMs. Total counts at each stage are provided (12.49M functions total, 384 sampled per year)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with The Hong Kong University of Science and Technology, clearly stated on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table I documents training data time spans for all eight CLMs (e.g., StarCoder: Jan 2015 - Mar 2022, ChatGPT-3.5: up to Sep 2021). This is central to the study's design."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about train/test overlap. The temporal splitting design explicitly addresses this, and Section IV-A validates low overlap using similarity metrics."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 directly addresses benchmark contamination by comparing HumanEval and CoderEval release dates against model cutoff dates (Section II-A2, III-D)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time reported despite running 8 models across multiple code groups with nucleus sampling."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (two NVIDIA RTX 6000 Ada GPUs) but total GPU hours or compute budget is not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "CLMs do not necessarily perform worse on recent data (post-cutoff); they sometimes perform better.",
       "evidence": "Table II shows Pass@k scores on Code-2023 and Code-2023-oct consistently outperform earlier code groups across all CLMs (Section IV-B).",
       "supported": "strong"
     },
     {
       "claim": "CLMs perform better on curated datasets (HumanEval, CoderEval) compared to contaminated data.",
       "evidence": "Table II shows Code-CodE and Code-HumE achieve higher Pass@k than Code-2018 through Code-2022 for most models (Section IV-C).",
       "supported": "strong"
     },
     {
       "claim": "Syntactic refactoring (IFF, Loop) does not reliably reduce CLM performance; loop transformation can even improve it.",
       "evidence": "Table III shows Code-Loop increases Pass@1 for all 8 models, while Code-IFF has mixed results (Section IV-D).",
       "supported": "strong"
     },
     {
       "claim": "Semantic refactoring operators (Renm, Param, Deco) are more effective at reducing CLM performance.",
       "evidence": "Table III shows Code-Deco consistently decreases Pass@1 across all models, with drops ranging from -1.9 to -15.9 (Section IV-D).",
       "supported": "strong"
     },
     {
       "claim": "Existing MIA-related metrics cannot distinguish contaminated from cleansed data.",
       "evidence": "Figure 6 shows similar metric scores across contaminated and cleansed code groups for StarCoder (Section IV-E).",
       "supported": "moderate"
     },
     {
       "claim": "AI programming assistants like Copilot may exacerbate data contamination.",
       "evidence": "Speculative explanation based on GitHub's report that 40% of code is written by Copilot. No direct evidence linking AI-generated code to the observed performance patterns (Section IV-B).",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "The paper systematically evaluates three countermeasures against data contamination in code language models: using recent data, curated benchmarks, and code refactoring. Counter-intuitively, CLMs often perform better on post-cutoff and curated data than on contaminated data, suggesting these countermeasures may not effectively mitigate contamination. Semantic refactoring operators (identifier renaming, parameter appending, decorator addition) show more promise in disrupting memorized patterns than syntactic operators. Existing membership inference metrics (perplexity, Zlib entropy, MIN-K% PROB) cannot reliably distinguish contaminated from clean data.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "Performance differences across code groups are compared using only point estimates without any significance testing, despite the study's central claims being about whether differences exist between contaminated and cleansed data."
     },
     {
       "flag": "No variance or error bars reported",
       "detail": "Despite using nucleus sampling with 5 samples per example, no run-level variance or confidence intervals are reported for Pass@k scores."
     },
     {
       "flag": "Speculative causal explanation",
       "detail": "The claim that AI coding assistants explain why CLMs perform better on Code-2023 is speculative. No evidence is provided to verify the proportion of AI-generated code in the dataset, and alternative explanations (evolving coding conventions, different repository quality filters for 2023) are not fully explored."
     },
     {
       "flag": "No artifacts released",
       "detail": "Neither code nor data is released, limiting reproducibility of a study about reproducibility concerns in CLM evaluation."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "relevance": "Introduces HumanEval benchmark, one of the most widely used code generation benchmarks and a key subject of contamination concerns."
     },
     {
       "title": "NLP evaluation in trouble: On the need to measure LLM data contamination for each benchmark",
       "authors": ["O. Sainz", "J. Campos", "I. García-Ferrero"],
       "year": 2023,
       "relevance": "Directly addresses data contamination measurement in LLM evaluation, a core concern for benchmark reliability."
     },
     {
       "title": "Detecting pretraining data from large language models",
       "authors": ["W. Shi", "A. Ajith", "M. Xia"],
       "year": 2023,
       "arxiv_id": "2310.16789",
       "relevance": "Proposes MIN-K% PROB for membership inference attacks on LLMs, one of the metrics evaluated in this paper."
     },
     {
       "title": "Extracting training data from large language models",
       "authors": ["N. Carlini", "F. Tramèr", "E. Wallace"],
       "year": 2021,
       "relevance": "Foundational work on training data extraction and memorization in LLMs, directly relevant to contamination concerns."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation",
       "authors": ["X. Du", "M. Liu", "K. Wang"],
       "year": 2023,
       "relevance": "Example of curated benchmark designed to reduce contamination risk in code generation evaluation."
     },
     {
       "title": "CoderEval: A benchmark of pragmatic code generation with generative pre-trained models",
       "authors": ["J. Yu"],
       "year": 2023,
       "relevance": "Curated coding benchmark used as one of the countermeasure datasets in this study."
     },
     {
       "title": "How effective are neural networks for fixing security vulnerabilities",
       "authors": ["Y. Wu", "N. Jiang", "H. V. Pham"],
       "year": 2023,
       "relevance": "Prior work that adopted identifier renaming and code structure change as contamination countermeasures."
     },
     {
       "title": "Large language models for software engineering: Survey and open problems",
       "authors": ["A. Fan", "B. Gokkaya", "M. Harman"],
       "year": 2023,
       "arxiv_id": "2310.03533",
       "relevance": "Survey of LLMs for SE that discusses using AI for code refactoring to address contamination."
     },
     {
       "title": "Time travel in LLMs: Tracing data contamination in large language models",
       "authors": ["S. Golchin", "M. Surdeanu"],
       "year": 2023,
       "arxiv_id": "2308.08493",
       "relevance": "Proposes methods for tracing data contamination in LLMs, directly relevant to benchmark reliability."
     },
     {
       "title": "Task contamination: Language models may not be few-shot anymore",
       "authors": ["C. Li", "J. Flanigan"],
       "year": 2023,
       "arxiv_id": "2312.16337",
       "relevance": "Studies task-level contamination in LLMs, a related but distinct contamination concern."
     }
   ]
 }