ai-research-survey

scan.json (31699B)

---

 {
   "paper": {
     "title": "DyCodeEval: Dynamic Benchmarking of Reasoning Capabilities in Code Large Language Models Under Data Contamination",
     "authors": [
       "Simin Chen",
       "Pranav Pusarla",
       "Baishakhi Ray"
     ],
     "year": 2025,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2503.04149",
     "doi": "10.48550/arXiv.2503.04149"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DyCodeEval generates semantically diverse yet complexity-equivalent programming problem variants using a four-agent pipeline, mitigating data contamination effects in Code LLM benchmarking. Controlled contamination experiments on 3 small models show static benchmarks inflate Pass@1 by up to 2-4x while dynamic benchmarks remain stable. In-the-wild evaluation of 18 models flags QWEN2.5-CODER-7B as potentially contaminated on both HumanEval and MBPP. The proposed DyPass@K metric is robust to contamination and produces consistent results across 10 random trials (low std dev).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a project webpage URL (https://codekaleidoscope.github.io/dycodeeval.html) but does not explicitly state that source code is released. The URL is a project page, not a repository link."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two publicly available seed datasets: HumanEval (Chen et al., 2021) and MBPP-Sanitized (Austin et al., 2021), both standard public benchmarks described in §4.1 and Appendix B."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using vLLM for open-source models and commercial APIs for closed-source models but provides no requirements.txt, Dockerfile, or detailed environment specifications beyond these tool mentions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The pipeline is described conceptually but there are no concrete commands or scripts to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The stability experiment (§4.5, Fig. 6) reports mean and standard deviation across 10 runs. The in-the-wild analysis (§4.3, Fig. 5) shows 95% confidence interval regression bands."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Comparative claims across models and conditions are made by comparing raw Pass@1 scores and visual regression analysis without formal hypothesis testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute Pass@1 scores at each contamination level (0%, 25%, 50%, 75%, 100%) providing baseline and treatment context. Tables 2-3 compare Pass@K and DyPass@K with specific values."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the choice of seed datasets (164 HumanEval, 427 MBPP problems), the number of models tested, or the number of scenarios/contexts (50 each). No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 and Fig. 6 explicitly report mean and standard deviation of Pass@1 scores across 10 generated benchmark datasets for the stability analysis."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares DyCodeEval against multiple baselines for diversity: Token Mutation, Char Mutation, Func Mutation, Insert Line, CommSyntax, and PPM. Static benchmarking serves as the baseline for contamination experiments."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include PPM (Chen et al., 2024) and ReCode mutations (Wang et al., 2023), which are recent methods. LiveCodeBench (Jain et al., 2024) is also discussed as a related approach."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has four distinct agents (Scenario Proposer, Context Generator, Prompt Rewriter, Validator) but no ablation study removes or modifies individual components to measure their contribution. Section 4.6 only tests swapping the foundation LLM."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Pass@1, Pass@K, BLEU-4, cosine semantic similarity, and the proposed DyPass@K metric across different evaluation dimensions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D describes a human verification step where two graduate-level students independently reviewed 60 randomly sampled problem pairs (30 per dataset) for consistency, achieving 95% agreement after discussion."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For the contamination experiment (§4.2), the contaminated model is evaluated on both the leaked benchmark and a separate non-leaked benchmark. The dynamic benchmarks are newly generated each time, ensuring no overlap with training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model (18 models), contamination level (0/25/50/75/100%), dataset (HumanEval vs MBPP), internal vs external diversity (Table 1), and Pass@K values (k=3,5,10 in Tables 2-3)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses the DEEPSEEK-CODER anomaly where 25% contamination drops performance below baseline (§4.2). The limitations section notes cases where generated problems contain excessive information that may confuse readers."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The consistency rate drops from 95% to 83% when using CLAUDE-3.5-HAIKU instead of SONNET as the foundation model (§4.6). The DEEPSEEK-CODER performance degradation at 25% contamination is an unexpected negative finding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's three main claims — effective assessment under contamination (§4.2), diverse problem generation (§4.4, Table 1), and robust/consistent benchmarking (§4.5, Fig. 6) — are all supported by corresponding experimental sections."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim ('data contamination creates a false sense of code reasoning capability') is supported by a controlled intervention design (§4.2) where the authors deliberately introduce contamination at varying levels and measure its effect on both static and dynamic benchmarks."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims results for 'Code Large Language Models' broadly, but experiments are limited to Python code generation on two benchmarks (HumanEval and MBPP). The contamination experiment uses only 3 small models (1B-3B parameters). No discussion of whether findings extend to other languages or larger models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper hypothesizes about the DEEPSEEK-CODER anomaly (§4.2) but does not systematically discuss alternative explanations for the main findings, such as whether benchmark difficulty differences could explain performance gaps, or whether the regression outlier detection could produce false positives."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures Pass@1 on code generation benchmarks but repeatedly claims to measure 'reasoning capabilities.' Pass@1 on HumanEval/MBPP is a proxy for reasoning ability, but the paper does not acknowledge this gap or discuss what reasoning capability actually entails beyond benchmark performance."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies model families and sizes (e.g., LLAMA-3.2-1B, DEEPSEEK-CODER-1.3B) but the critical foundation model 'CLAUDE-3.5-SONNET' used for benchmark generation lacks a version date or API snapshot identifier. Multiple Claude-3.5-Sonnet versions exist with different behaviors."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides full prompt templates for all four agents (Scenario Proposer, Context Generator, Prompt Rewriter, and two Validation prompts) along with worked examples showing filled-in values. A reader could reconstruct the prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: temperature 0.8 for generation, temperature 0 for validation. The number of scenarios (50) and contexts per scenario (50) are also specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The four-agent pipeline (Scenario Proposer, Context Generator, Prompt Rewriter, Validator) is described in detail in §3.2 with a workflow diagram (Fig. 2), inter-agent data flow, type inference algorithm (Alg. 1), and retry logic when validation fails."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B describes how MBPP-Sanitized was refined by 'adding function headers and converting natural language instructions into function docstrings.' The contamination fine-tuning process and data splits (0/25/50/75/100%) are documented in §4.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Conclusion) contains a substantive limitations paragraph identifying two specific limitations: (1) computational cost requiring large LLMs for high consistency rates, and (2) excessive information in generated prompts that may confuse readers."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: the consistency rate drops from 95% to 83% with a smaller foundation LLM (§4.6), and generated questions sometimes contain excessive information. These are concrete, study-specific observations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no discussion of limitations to specific languages (Python only), model sizes (only 1B-13B for contamination experiments), or task types (only NL-to-code generation)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The generated benchmark datasets, fine-tuned model weights, and raw evaluation outputs are not provided for independent verification. Only aggregate results (Pass@1 scores, diversity metrics) are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark generation pipeline is described in detail (§3.2). Seed datasets are described in Appendix B. The contamination fine-tuning process is documented in §4.2 with specific percentages."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The human verification (Appendix D) mentions 'two graduate-level students' but provides no details about their qualifications, how they were selected, or potential biases in their evaluation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The four-stage pipeline is documented with the type inference algorithm (Alg. 1), prompt templates (Appendix C), validation criteria (two-step validation), and the iterative retry process. The 95% human agreement rate provides a quality metric."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section states: 'This work was supported in part by CCF 2313055, CCF 2107405, CAREER 2025082, and FAI: 2040961.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are disclosed as being from Columbia University Department of Computer Science. They evaluate models from external companies (Meta, DeepSeek, Alibaba, Anthropic) and use Anthropic's Claude as their foundation model."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from NSF grants (CCF, CAREER, FAI programs), which are government research grants independent of any LLM vendor's commercial interests."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the 18 evaluated models, despite the paper being specifically about data contamination. This makes it difficult to assess the baseline contamination status of the in-the-wild models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about train/test overlap. Section 4.2 creates controlled contamination experiments, and §4.3 uses regression analysis to detect potential contamination in wild models (flagging QWEN2.5-CODER-7B as an outlier)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark contamination is the paper's core concern. It explicitly discusses how HumanEval and MBPP are likely in many models' training data and proposes dynamic generation as a mitigation strategy."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper with no human participants in the research sense. The two annotators in Appendix D perform validation, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. The paper evaluates Code LLMs on programming benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study. The two annotators in Appendix D are described only as 'graduate-level students.'"
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Computational cost is mentioned as a limitation in §6 but no actual costs, token counts, API spend, or wall-clock times are reported. Section 4.6 discusses using a cheaper model but does not quantify the savings."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total API spend, or hardware specifications are reported despite the method requiring extensive LLM calls (Claude-3.5-Sonnet for generating 50 scenarios × 50 contexts × hundreds of problems plus validation)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 runs DyCodeEval 10 times and reports mean and standard deviation of Pass@1 scores, demonstrating sensitivity to the randomness in the generation process."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 explicitly states 'we run DyCodeEval 10 times and measure the Pass@1 scores across these 10 generated benchmark datasets.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The choice of 50 scenarios, 50 contexts, temperature 0.8, and temperature 0 for validation appear to be fixed without justification or search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explain how the configuration (50 scenarios, 50 contexts, temperature settings) was selected. No validation-based selection or sensitivity analysis across configurations is provided."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 18 models across two datasets with multiple metrics but performs no statistical tests at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose DyCodeEval and evaluate it against baselines they implemented, without acknowledging author-evaluation bias or having independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The dynamic benchmark generation requires extensive LLM calls (Claude-3.5-Sonnet) compared to static benchmarking, but this compute difference is not quantified or discussed in relation to the evaluation benefits."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central thesis addresses construct validity: static benchmarks may measure memorization rather than reasoning under contamination. The dynamic approach is explicitly designed to test whether models are 'genuinely reasoning to solve it' (§5)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The evaluated Code LLMs are tested via straightforward prompt-to-code generation without scaffolding. No agentic scaffolding is involved in the model evaluation step."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core contribution addresses temporal leakage: static benchmarks published before model training can leak into training data. DyCodeEval generates fresh variants to avoid this. Section 1 explicitly discusses this concern."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the evaluation setup leaks answer information through context. The generated prompts provide rich contextual information that could potentially provide hints not available in the original problems."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All generated problems derive from the same seed problems (HumanEval and MBPP), sharing underlying algorithmic structure. The paper does not discuss whether this structural non-independence affects the validity of using generated problems as independent test items."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 uses regression analysis with 95% confidence intervals to detect potential contamination in wild models (flagging outliers like QWEN2.5-CODER-7B). The collision analysis (§3.3, Theorems 3.1-3.3) provides theoretical bounds on the probability of generating identical problems."
       }
     }
   },
   "claims": [
     {
       "claim": "Data contamination creates a false sense of code reasoning capability under static benchmarks, with contaminated models achieving inflated Pass@1 scores on leaked benchmarks.",
       "evidence": "Section 4.2, Fig. 4: Fine-tuned models with 100% leaked data show substantially higher Pass@1 on contaminated benchmarks but not on non-contaminated ones. Three models tested at 0/25/50/75/100% contamination levels.",
       "supported": "strong"
     },
     {
       "claim": "DyCodeEval's dynamic benchmarking mitigates the impact of data contamination, preventing artificially inflated scores.",
       "evidence": "Section 4.2, Fig. 4 (second row): Dynamic benchmark scores remain stable regardless of contamination level, unlike static benchmarks. Demonstrated on LLAMA-3.2-1B, LLAMA-3.2-3B, and DEEPSEEK-CODER-1.3B.",
       "supported": "moderate"
     },
     {
       "claim": "QWEN2.5-CODER-7B is potentially contaminated on both HumanEval and MBPP datasets.",
       "evidence": "Section 4.3, Fig. 5: QWEN2.5-CODER-7B consistently falls outside the 95% confidence interval of the in-the-wild regression area for both datasets.",
       "supported": "weak"
     },
     {
       "claim": "DyCodeEval generates semantically diverse programming problems, significantly outperforming baseline mutation methods.",
       "evidence": "Section 4.4, Table 1: DyCodeEval achieves BLEU-4 of 0.17-0.27 (internal) and 0.02-0.17 (external) compared to baselines ranging 0.57-1.00. Semantic similarity drops to 0.59-0.74 vs 0.84-1.00 for baselines.",
       "supported": "strong"
     },
     {
       "claim": "DyCodeEval produces stable benchmarking results despite its inherent randomness.",
       "evidence": "Section 4.5, Fig. 6: Variance in Pass@1 scores is minimal across 10 independent runs, with standard deviations much smaller than mean values.",
       "supported": "strong"
     },
     {
       "claim": "DyPass@K is more robust to contamination than Pass@K, accurately reflecting reasoning capability even under data leakage.",
       "evidence": "Section 5, Tables 2-3: For contaminated LLAMA-3.2-1B, Pass@3 jumps from 0.22 to 0.82 while DyPass@3 drops from 0.17 to 0.13. Non-contaminated models show consistent scores across both metrics.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Contamination experiment limited to tiny models",
       "detail": "The controlled contamination experiment (§4.2) uses only 3 models of 1B-3B parameters. These very small models may be more susceptible to overfitting on leaked data than larger, more capable models, limiting the generalizability of the contamination findings."
     },
     {
       "flag": "Contamination accusation without ground truth",
       "detail": "The paper claims QWEN2.5-CODER-7B is 'potentially contaminated' based solely on being a regression outlier (§4.3). There is no independent verification, no access to Qwen's training data, and no discussion of alternative explanations for the outlier behavior (e.g., different training objectives, data curation strategies)."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims across 18 models and multiple conditions rely on raw score comparisons without any formal statistical testing. Differences may not be statistically meaningful, especially given the variance shown in §4.5."
     },
     {
       "flag": "Very small human evaluation sample",
       "detail": "The human verification (Appendix D) uses only 2 annotators reviewing 60 problem pairs. This is a minimal sample for validating an entire automated benchmarking approach, and inter-annotator agreement before discussion is not reported separately."
     },
     {
       "flag": "Foundation model evaluates its own outputs",
       "detail": "Claude-3.5-Sonnet serves as both the generation engine (creating benchmark problems) and the validation agent (checking consistency). Self-validation by the same model family may miss systematic biases in the generation."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, one of the most widely used code generation benchmarks and a seed dataset in this study."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Proposes contamination-free code benchmarking through continuous collection of new problems, a key baseline approach this paper aims to improve upon."
     },
     {
       "title": "PPM: Automated generation of diverse programming problems for benchmarking code generation models",
       "authors": ["Simin Chen", "Xin Feng", "Xinyu Han"],
       "year": 2024,
       "relevance": "Prior work by the same first author on automated benchmark generation using manually defined operators, used as a diversity comparison baseline."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Introduces EvalPlus, identifying limitations of HumanEval/MBPP test coverage and proposing more rigorous code generation evaluation."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Major real-world code benchmark for evaluating LLMs on GitHub issue resolution, relevant to benchmarking methodology and contamination concerns."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming—the rise of code intelligence",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Describes the DeepSeek-Coder model family evaluated in this study, relevant to code LLM development and capability assessment."
     },
     {
       "title": "ReCode: Robustness evaluation of code generation models",
       "authors": ["Shiqi Wang", "Zheng Li", "Haifeng Qian"],
       "year": 2023,
       "doi": "10.18653/V1/2023.ACL-LONG.773",
       "relevance": "Proposes robustness-based mutations for code generation evaluation, used as a baseline for diversity comparison in this study."
     },
     {
       "title": "DyVal: Dynamic evaluation of large language models for reasoning tasks",
       "authors": ["Kaijie Zhu", "Jiaao Chen", "Jindong Wang"],
       "year": 2024,
       "arxiv_id": "2309.17167",
       "relevance": "Uses DAG structures for dynamic LLM benchmarking, a key related approach for contamination-resistant evaluation."
     },
     {
       "title": "Recent advances in large language model benchmarks against data contamination: From static to dynamic evaluation",
       "authors": ["Simin Chen", "Yanxin Chen", "Zelin Li"],
       "year": 2025,
       "arxiv_id": "2502.17521",
       "relevance": "Survey of contamination-free benchmarking approaches by the same first author, providing context on the contamination problem landscape."
     },
     {
       "title": "Generalization or memorization: Data contamination and trustworthy evaluation for large language models",
       "authors": ["Yicheng Dong", "Xiangjue Jiang", "Hanlin Liu"],
       "year": 2024,
       "arxiv_id": "2402.15938",
       "relevance": "Studies data contamination effects on LLM evaluation trustworthiness, directly relevant to understanding contamination's impact on benchmarking."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper that first raised data contamination concerns in LLM benchmarking at scale."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell I. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces the MBPP benchmark used as one of two seed datasets in this study."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "DyCodeEval provides a usable framework for contamination-resistant benchmarking, though setup requires an LLM agent pipeline."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Data contamination in benchmarks is a known concern; the specific finding about QWEN2.5-CODER-7B adds a mildly surprising result."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Raises concerns about unreliable LLM evaluations due to contamination but does not demonstrate novel attacks or safety threats."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Publicly flagging QWEN2.5-CODER-7B as potentially contaminated and arguing that 'static benchmarks create a false sense of accuracy' has moderate controversy potential."
     },
     "demo_ability": {
       "score": 1,
       "justification": "A project webpage exists but no readily installable tool or live demo is provided in the paper."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Columbia University authors, published at ICML. Evaluates well-known models but is not from a major AI lab."
     }
   }
 }