ai-research-survey

scan.json (31686B)

---

 {
   "paper": {
     "title": "Quantifying Contamination in Evaluating Code Generation Capabilities of Language Models",
     "authors": ["Martin Riddell", "Ansong Ni", "Arman Cohan"],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2403.04811",
     "doi": "10.48550/arXiv.2403.04811"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "Between 3.6% and 20.8% of MBPP and HumanEval benchmark solutions appear in the PILE and STACK pretraining corpora via surface-level and semantic-level matching. Models perform dramatically better on contaminated subsets (up to 50+ percentage point gap between top-10% and bottom-10% similarity bins). De-contaminated accuracy drops by up to 82.7% for some model-benchmark pairs, and much of the apparent performance gap between model families is attributable to differential contamination rather than capability differences.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link in footnote 1: 'Code and data available at https://github.com/yale-nlp/code-llm-contamination' and states 'We release all resulting files from our matching pipeline for future research.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The matching results are released via the same GitHub link. Additionally, the benchmarks (MBPP, HumanEval) and pretraining corpora (PILE, STACK) are publicly available datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification (requirements.txt, Dockerfile, conda environment) is mentioned in the paper. They reference the rapidfuzz library (footnote 2) and Dolos toolkit, but no comprehensive dependency list is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The methodology is described at a high level but lacks specific commands or scripts to replicate the matching pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-4 are reported as point estimates without confidence intervals or error bars. No uncertainty quantification is provided for any accuracy or similarity score."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims models perform 'significantly better' on contaminated subsets but provides no statistical significance tests (no p-values, t-tests, or other formal tests). Comparisons are based solely on numerical differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with context: performance gaps with baselines (e.g., Tab 2 shows 72% vs 22% accuracy for top/bottom 10%), relative accuracy degradation percentages in Tab 1 (e.g., '-82.7%'), and the absolute performance gap metric (∆⇕)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes. The 500 MBPP test questions and 164 HumanEval questions are used as-is without power analysis or discussion of whether these sizes support the claimed statistical patterns."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported. Model outputs are taken from L2CEval as single-run results. No multiple-seed analysis or spread measures are provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares multiple model families (Pythia, CodeGen-NL, StarCoderBase) across multiple sizes, and compares contaminated vs. de-contaminated performance. Original accuracy serves as the baseline for measuring contamination effects."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models studied (StarCoderBase, Pythia, CodeGen-NL) were current open-source models at the time of writing. The selection criteria are clearly justified: models must have publicly available training data."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates on surface-level vs. semantic-level similarity (Fig 3, Fig 11), different similarity thresholds (Tab 1 with scores of 100, >90, >80), model sizes (Fig 5, Tab 4), and contamination subsets vs. complements (Tab 3a)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: surface-level Levenshtein similarity, semantic-level Dolos similarity, aggregated similarity scores, top-1 and top-10 scores, pass@1 accuracy, de-contaminated accuracy, and performance gap (∆⇕)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. The similarity matching and accuracy evaluation are entirely automated. Manual inspection is only mentioned for threshold justification (footnote 5) but not as a formal evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on the standard test splits of MBPP (500 questions) and HumanEval (164 questions). Model outputs are from L2CEval's standard evaluation protocol."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns are provided: per-model (12 models), per-benchmark (MBPP vs HumanEval), per-threshold, per-similarity-bin (Fig 5), per-subset seen/unseen (Tab 3a), and per-corpus (PILE vs STACK)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 presents a case study of examples where models fail despite having seen similar solutions 10+ times in training, with two representative examples (Fig 7, Fig 8) and discussion of why failure occurs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3a shows that Pythia and CodeGen-NL models actually perform worse on the STACK-seen subset than on unseen questions (e.g., CodeGen-NL-16B: 11.5% on seen vs. 21.7% on unseen for MBPP). The case study also reports models failing on highly contaminated examples."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'substantial overlap' are supported by Fig 1 and Section 4.1 (3.6-20.8% exact matches). Claims of 'significantly better' performance on contaminated subsets are supported by Tab 2 (50+ point gaps). Analysis of model size, difficulty, and question length are in Section 4.2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims ('models perform significantly better when similar solutions are seen during training') from observational data. While Section 4.2 attempts to decouple memorization and difficulty by cross-referencing subsets across model families, this does not establish causation. Confounds such as problem type, coding style, or algorithmic patterns shared between training data and benchmarks are not controlled for."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds its claims to MBPP and HumanEval benchmarks, three model series, and two corpora. The Limitations section (§6) acknowledges that only gold solutions were searched, compute constraints limited the search scope, and more models/benchmarks are needed."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 explicitly addresses the alternative explanation that contaminated questions might simply be easier, using cross-model subset analysis to show this is not the case. The effect of program length is analyzed in Fig 6 to rule out length as a confound."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to distinguish between its similarity scores (proxy) and actual contamination (outcome). It acknowledges in Limitations that searching only gold solutions provides a 'minimum number of questions' exposed, that false positives exist in semantic matching, and that multiple correct solutions mean their measurements are lower bounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model names and sizes are specified: Pythia-1.4B/2.8B/6.9B/12B, CodeGen-NL-350M/2B/6B/16B, StarCoderBase-1B/3B/7B/15.5B (Section 3.1). These are specific named model checkpoints from known training runs."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses model outputs from L2CEval (Ni et al., 2023) but does not provide the exact prompt formats used for benchmark evaluation. The reader is directed to L2CEval without the actual prompt text being included."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No generation hyperparameters (temperature, top-p, max tokens) are reported for the model evaluations. The outputs are taken from L2CEval without specifying these settings. For the matching pipeline, the top-500 threshold is justified (footnote 5) but other parameters (sliding window step size, etc.) are not specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates base models directly on benchmarks and runs a similarity matching pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is well documented in Section 2: sliding window character-by-character scan, Levenshtein similarity computation, top-500 selection, Dolos semantic matching, and aggregation via max(Ssurface, Ssemantic). Section 3.1 specifies the GitHub split (95.16 GiB) for PILE and 60.40 GB Python split for STACK."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations' provides substantive discussion across four specific subsections: multiple correct solutions, compute costs limiting search, false positives, and scarcity of open data."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: searching only gold solutions means contamination is a lower bound, compute constraints limited search to GitHub/Python splits only, Dolos semantic matching can produce false positives (with examples referenced in §A.4), and only two benchmarks with gold solutions could be studied."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it covers only MBPP and HumanEval, only three model series with open training data, only gold solutions (not all correct solutions), and only specific splits of training data. It notes 'We hope more open models with open data will become available in the future.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper releases all matching pipeline results via GitHub (footnote 1). The underlying benchmarks (MBPP, HumanEval) and training corpora (PILE, STACK) are publicly available. Appendix A.4 provides lists of all perfect matches."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the matching pipeline in detail: character-by-character sliding window for surface matching, tree-sitter AST tokenization for semantic matching via Dolos, and aggregation. Section 3 describes model selection criteria and data sources."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses publicly available benchmarks and pretraining corpora."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: benchmark gold solutions → sliding window Levenshtein matching over training data → top 500 surface matches per question → Dolos semantic matching → aggregated score = max(surface, semantic). Thresholds and filtering criteria are specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The Acknowledgements section thanks specific individuals but does not mention any funding sources, grants, or financial support."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are disclosed as being from the Department of Computer Science, Yale University. They are not affiliated with any of the companies whose models they evaluate."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The authors are academic researchers with no apparent financial stake in the models evaluated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Rather than stating cutoff dates, the paper goes further: it uses the actual training data (PILE GitHub split, STACK Python split) and directly searches through it. The training datasets are fully specified and publicly available (Section 3.1)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Train/test overlap is the central focus of the paper. Section 4.1 precisely quantifies the overlap using both surface-level and semantic-level matching, finding 3.6-20.8% exact matches."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark contamination is the paper's primary contribution. They note that STACK went through 'a string-matching-based decontamination process for MBPP and HumanEval, but we are still able to find traces of contamination' (footnote 7), demonstrating that standard decontamination is insufficient."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a computational analysis of training data overlap."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyzes publicly available datasets and models."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported for the model evaluations. Footnote 4 estimates '5.2×10^5 CPU hours to search just the Python files from the STACK for MBPP' but this is used to justify search limitations, not as systematic cost reporting."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The total computational budget for the matching pipeline is not stated. Footnote 4 provides a theoretical estimate for full search, and they mention limiting to specific splits to reduce cost, but the actual compute spent is not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. Model outputs are taken as single runs from L2CEval. The matching pipeline is deterministic, but model generation is not."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation runs is not stated. Results appear to be from single-run evaluations via L2CEval."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No formal hyperparameter search budget is reported. The top-500 threshold is justified by manual inspection (footnote 5: '95% of them have a similarity score < 72'), but the overall search budget for pipeline parameters is not disclosed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports results at multiple thresholds (100, >90, >80) rather than selecting only the most favorable one. The aggregation method (max of surface and semantic) is motivated by the complementary examples in Figs 2 and 4."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparisons across models, benchmarks, thresholds, and subsets without any correction for multiple comparisons. No formal statistical tests are used at all, let alone corrected ones."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors do not discuss potential biases in their own evaluation methodology. While they acknowledge false positives in the Limitations section, they do not discuss how their pipeline design choices might systematically bias contamination estimates."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not propose a competing method where compute budget vs. performance tradeoffs are relevant. It is an analysis study measuring contamination."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper questions whether MBPP and HumanEval benchmark scores actually measure model capability vs. memorization. This is a direct investigation of construct validity for code generation benchmarks."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated directly on benchmarks without agentic scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly addresses temporal leakage by searching pretraining corpora for benchmark solutions. They note that MBPP and HumanEval predate the training data collection, creating contamination risk."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the benchmark evaluation setup itself leaks information (e.g., whether docstrings or test cases in HumanEval prompts contain answer hints). The focus is entirely on training data overlap."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures non-independence between training and test data through exhaustive similarity matching. Tables 3b and 5-13 provide exact counts and examples of overlapping instances."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper proposes and applies a concrete two-stage detection method: Levenshtein surface-level matching via sliding window, followed by Dolos AST-based semantic matching. This is the paper's core contribution."
       }
     }
   },
   "claims": [
     {
       "claim": "3.6% to 20.8% of MBPP and HumanEval benchmark solutions are found in pretraining corpora as exact matches",
       "evidence": "Fig 1 and Section 4.1: 12.2% of HumanEval in PILE, 18.9% in STACK; 3.6% of MBPP in PILE, 20.8% in STACK (using aggregated similarity score = 100).",
       "supported": "strong"
     },
     {
       "claim": "Models perform significantly better on questions where similar solutions were seen during training",
       "evidence": "Table 2: StarCoderBase-15.5B shows 72% accuracy on top-10% similarity questions vs. 22% on bottom-10% for MBPP (50pp gap). Pythia-12B shows 56.3% vs 0% on HumanEval (56.3pp gap).",
       "supported": "strong"
     },
     {
       "claim": "Removing contaminated examples causes substantial accuracy drops, and narrows performance gaps between model families",
       "evidence": "Table 1: Pythia-12B accuracy drops from 9.8% to 1.7% (-82.7%) on HumanEval at threshold >80. The StarCoderBase-Pythia gap shrinks from 23.8% to 13.9% after de-contamination.",
       "supported": "strong"
     },
     {
       "claim": "Larger models are better at both memorization and generalization",
       "evidence": "Figure 5 shows across all three model families that larger models consistently achieve higher accuracy at every similarity bin, indicating improvement in both exploiting seen data and solving unseen problems.",
       "supported": "moderate"
     },
     {
       "claim": "The performance advantage on contaminated examples is not explained by question difficulty",
       "evidence": "Table 3a: CodeGen-NL-16B has 19.6% overall accuracy but only 11.5% on the 104 MBPP questions seen by StarCoderBase, indicating these questions are not inherently easier. Models not trained on STACK perform worse on the STACK-seen subset.",
       "supported": "moderate"
     },
     {
       "claim": "STACK's string-matching decontamination process was insufficient for removing benchmark overlap",
       "evidence": "Footnote 7 notes the STACK underwent decontamination, yet the study still finds 20.8% of MBPP and 18.9% of HumanEval solutions in STACK, demonstrating semantic-level contamination that surface decontamination missed.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "The paper uses the word 'significantly' when describing performance differences but provides no formal statistical tests. All comparisons are based on raw numerical differences without p-values, confidence intervals, or any inferential statistics."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All accuracy results are single-run point estimates from L2CEval with no error bars, standard deviations, or confidence intervals. The stability of the reported performance gaps across different evaluation conditions is unknown."
     },
     {
       "flag": "Causal claims from observational data",
       "detail": "The paper implies contamination causes better performance, but the study design is observational. While the cross-model difficulty analysis (Section 4.2) is clever, it does not fully control for confounds such as shared algorithmic patterns or coding style correlations."
     },
     {
       "flag": "Limited search scope understates contamination",
       "detail": "The search was limited to the GitHub split of PILE and the Python split of STACK due to compute constraints. Similar solutions may exist in other programming languages or non-code text, meaning the reported contamination rates are lower bounds, but the magnitude of the underestimate is unknown."
     }
   ],
   "cited_papers": [
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye", "Maarten Bosma", "Henryk Michalewski", "David Dohan", "Ellen Jiang", "Carrie Cai", "Michael Terry", "Quoc Le", "Charles Sutton"],
       "year": 2021,
       "relevance": "Introduces MBPP, one of the two primary benchmarks studied for contamination in this paper."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "relevance": "Introduces HumanEval, the other primary code generation benchmark studied for contamination."
     },
     {
       "title": "Starcoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "relevance": "StarCoderBase models are a primary subject of the contamination study, trained on the STACK corpus."
     },
     {
       "title": "Pythia: A suite for analyzing large language models across training and scaling",
       "authors": ["Stella Biderman", "Hailey Schoelkopf"],
       "year": 2023,
       "relevance": "Pythia model suite is one of three model families studied for training data contamination effects."
     },
     {
       "title": "Codegen: An open large language model for code with multi-turn program synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang"],
       "year": 2023,
       "relevance": "CodeGen-NL model series is studied for contamination effects on code generation benchmarks."
     },
     {
       "title": "The pile: An 800gb dataset of diverse text for language modeling",
       "authors": ["Leo Gao", "Stella Biderman"],
       "year": 2020,
       "relevance": "The PILE pretraining corpus is one of two training datasets searched for benchmark contamination."
     },
     {
       "title": "The stack: 3 tb of permissively licensed source code",
       "authors": ["Denis Kocetkov", "Raymond Li"],
       "year": 2022,
       "relevance": "The STACK pretraining corpus is the other training dataset searched for benchmark contamination."
     },
     {
       "title": "Quantifying memorization across neural language models",
       "authors": ["Nicholas Carlini", "Daphne Ippolito"],
       "year": 2023,
       "relevance": "Studies memorization in language models, directly relevant to understanding LLM training data leakage."
     },
     {
       "title": "Detecting pretraining data from large language models",
       "authors": ["Weijia Shi", "Anirudh Ajith", "Mengzhou Xia"],
       "year": 2023,
       "relevance": "Proposes methods for detecting contamination in black-box models using output probabilities."
     },
     {
       "title": "Time travel in llms: Tracing data contamination in large language models",
       "authors": ["Shahriar Golchin", "Mihai Surdeanu"],
       "year": 2023,
       "relevance": "Studies data contamination in LLMs for natural language tasks, complementary to this paper's code focus."
     },
     {
       "title": "L2ceval: Evaluating language-to-code generation capabilities of large language models",
       "authors": ["Ansong Ni", "Pengcheng Yin", "Yilun Zhao", "Martin Riddell"],
       "year": 2023,
       "arxiv_id": "2309.17446",
       "relevance": "Evaluation framework and model outputs used in this paper to obtain code generation performance numbers."
     },
     {
       "title": "Generalization or memorization: Data contamination and trustworthy evaluation for large language models",
       "authors": ["Yihong Dong", "Xue Jiang", "Huanyu Liu", "Zhi Jin", "Ge Li"],
       "year": 2024,
       "arxiv_id": "2402.15938",
       "relevance": "Identifies contamination through output distribution peakness for black-box LLMs, a complementary detection approach."
     },
     {
       "title": "Codex hacks hackerrank: Memorization issues and a framework for code synthesis evaluation",
       "authors": ["Anjan Karmakar", "Julian Aron Prenner", "Marco D'Ambros", "Romain Robbes"],
       "year": 2022,
       "relevance": "Demonstrates memorization patterns in code LLMs and proposes evaluation frameworks for code synthesis."
     },
     {
       "title": "CodeIPPrompt: Intellectual property infringement assessment of code language models",
       "authors": ["Zhiyuan Yu", "Yuhao Wu", "Ning Zhang"],
       "year": 2023,
       "relevance": "Uses Dolos plagiarism detection tool (also used in this paper) to assess code LLM intellectual property issues."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners evaluating LLMs on code benchmarks should know about contamination rates; the released pipeline could be used for contamination checking."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Quantifies that up to 20.8% of popular benchmark solutions appear in training data and that STACK's decontamination process was insufficient, challenging face-value benchmark results."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Raises concerns about reliability of LLM evaluations but does not demonstrate a novel security vulnerability or existential risk."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Directly challenges the validity of widely-cited benchmark results (MBPP, HumanEval) and shows that reported model performance gaps may be largely due to contamination."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code and data are released on GitHub, but the pipeline requires significant compute to run and is not a simple tool someone can quickly try."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Yale NLP is a respected academic group; the paper evaluates well-known models (StarCoder, Pythia) and benchmarks (HumanEval, MBPP) but is not from a top-tier industry lab."
     }
   }
 }