ai-research-survey

scan-v4.json (20354B)

---

 {
   "scan_version": 4,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "Dynamic Benchmarking of Reasoning Capabilities in Code Large Language Models Under Data Contamination",
     "authors": [
       "Simin Chen",
       "Pranav Pusarla",
       "Baishakhi Ray"
     ],
     "year": 2025,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2503.04149",
     "doi": "10.48550/arXiv.2503.04149"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's three main claims — effective assessment under contamination (§4.2), diverse problem generation (§4.4, Table 1), and robust/consistent benchmarking (§4.5, Fig. 6) — are all supported by corresponding experimental sections.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim ('data contamination creates a false sense of code reasoning capability') is supported by a controlled intervention design (§4.2) where the authors deliberately introduce contamination at varying levels and measure its effect on both static and dynamic benchmarks.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims results for 'Code Large Language Models' broadly, but experiments are limited to Python code generation on two benchmarks (HumanEval and MBPP). The contamination experiment uses only 3 small models (1B-3B parameters). No discussion of whether findings extend to other languages or larger models.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper hypothesizes about the DEEPSEEK-CODER anomaly (§4.2) but does not systematically discuss alternative explanations for the main findings, such as whether benchmark difficulty differences could explain performance gaps, or whether the regression outlier detection could produce false positives.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures Pass@1 on code generation benchmarks but repeatedly claims to measure 'reasoning capabilities.' Pass@1 on HumanEval/MBPP is a proxy for reasoning ability, but the paper does not acknowledge this gap or discuss what reasoning capability actually entails beyond benchmark performance.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Conclusion) contains a substantive limitations paragraph identifying two specific limitations: (1) computational cost requiring large LLMs for high consistency rates, and (2) excessive information in generated prompts that may confuse readers.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: the consistency rate drops from 95% to 83% with a smaller foundation LLM (§4.6), and generated questions sometimes contain excessive information. These are concrete, study-specific observations.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no discussion of limitations to specific languages (Python only), model sizes (only 1B-13B for contamination experiments), or task types (only NL-to-code generation).",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section states: 'This work was supported in part by CCF 2313055, CCF 2107405, CAREER 2025082, and FAI: 2040961.'",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are disclosed as being from Columbia University Department of Computer Science. They evaluate models from external companies (Meta, DeepSeek, Alibaba, Anthropic) and use Anthropic's Claude as their foundation model.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from NSF grants (CCF, CAREER, FAI programs), which are government research grants independent of any LLM vendor's commercial interests.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": false,
         "justification": "'Reasoning capabilities' is central to the paper's title and claims but is never precisely defined or distinguished from memorization beyond qualitative framing; 'semantic equivalence' is used extensively but defined only through examples rather than formally.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly enumerates three contributions: novel problem characterization, new methodology design (DyCodeEval), and empirical findings; readers know exactly what the paper adds.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 systematically covers three research lines (benchmarking Code LLMs, contamination-free strategies, LLM-as-judge) and situates DyCodeEval explicitly against PPM, LiveCodeBench, DyVal, and ITD.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.1 argues via metamorphic testing principles that modifying 'complexity-unrelated context description' while preserving 'complexity-related algorithmic abstraction' isolates reasoning from memorization; the theoretical grounding is explicit.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "The paper claims difficulty is preserved from seed datasets but provides no empirical analysis of the difficulty distribution of generated problems — no easy/medium/hard tiers and no difficulty measurements beyond aggregate Pass@1.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "No explicit ceiling/floor analysis is performed on the generated benchmark; the stability experiment (Fig 6) shows consistent means but does not report whether score distributions cluster near 0 or 1.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline is provided; the benchmark is evaluated only against LLMs with no measurement of human performance on either seed or generated problems.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Section 5 introduces DyPass with explicit justification for its superiority over Pass@K for contamination detection; Tables 2–3 empirically validate the metric's discriminative ability.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "Contamination resistance is the core design goal; dynamic generation with randomized scenario/context selection, theoretical collision bounds (Theorems 3.1–3.3), and a validation agent all explicitly target resistance to data leakage.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether DyCodeEval could be gamed if future LLMs train on its outputs, whether the scenario pool will become exhausted, or whether there is a plan for benchmark updates over time.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Only engineering limitations are mentioned (compute cost, excessive information); no discussion of benchmark-level failure modes such as systematic validation agent errors, canonical solution mismatches after rewriting, or scenario pool exhaustion.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": false,
           "justification": "Only a project webpage URL is mentioned; no explicit statement of code release, dataset availability, or reproducibility package is provided, making it unclear whether others can replicate reported numbers.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": false,
           "justification": "No data card or formal documentation of the DyCodeEval-generated benchmark is provided; Appendix B describes only the seed datasets (HumanEval, MBPP) and not the generated variants.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "No license or access terms are specified; only a project webpage URL is mentioned without terms of use or distribution rights.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": false,
           "justification": "The Impact Statement describes intended use in general terms but does not specify what should NOT be concluded from DyCodeEval results or explicit boundaries on valid use cases.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Data contamination creates a false sense of code reasoning capability under static benchmarks, inflating Pass@1 scores",
       "evidence": "Section 4.2 shows models fine-tuned on leaked HumanEval or MBPP data achieve higher Pass@1 on contaminated benchmarks while performance on uncontaminated benchmarks remains stable across contamination levels",
       "supported": "strong"
     },
     {
       "claim": "DyCodeEval prevents contaminated models from achieving artificially inflated Pass@1 scores through dynamic generation",
       "evidence": "Fig 4 shows the dynamic benchmarking rows do not exhibit the Pass@1 inflation seen in static rows for all three contamination-tested models",
       "supported": "strong"
     },
     {
       "claim": "DyCodeEval generates more syntactically and semantically diverse problems than existing mutation-based approaches",
       "evidence": "Table 1 shows DyCodeEval achieves substantially lower BLEU-4 (0.27/0.18 internal, 0.17/0.02 external) and semantic similarity compared to all baselines including PPM and character/token mutations",
       "supported": "strong"
     },
     {
       "claim": "DyCodeEval produces stable benchmarking results despite randomness",
       "evidence": "Fig 6 shows 10-run experiment with minimal variance in Pass@1 scores, though numeric standard deviations are not explicitly tabulated",
       "supported": "moderate"
     },
     {
       "claim": "QWEN2.5-CODER-7B is potentially contaminated on both HumanEval and MBPP",
       "evidence": "Fig 5 shows QWEN2.5-CODER-7B consistently falls outside the 95% confidence interval of the in-the-wild model regression area on both datasets",
       "supported": "moderate"
     },
     {
       "claim": "Using CLAUDE-3.5-HAIKU as foundation model reduces problem consistency rate from 95% to 83%",
       "evidence": "Section 4.6 reports manual sampling assessment of generated problems from each model, but sample size and methodology are not fully detailed",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "DyCodeEval combats data contamination in Code LLM benchmarking by dynamically generating semantically equivalent yet contextually varied Python programming problems using four LLM agents (Scenario Proposer, Context Generator, Prompt Rewriter, Validator) grounded in metamorphic testing principles. Controlled contamination experiments show that models fine-tuned on leaked benchmark data achieve inflated Pass@1 scores on static benchmarks but not on DyCodeEval-generated variants. Generated problems are substantially more diverse than mutation-based baselines (BLEU-4 as low as 0.02 vs 0.57–1.00 for alternatives). QWEN2.5-CODER-7B was flagged as likely contaminated on both HumanEval and MBPP based on its outlier position in regression analysis of 15 in-the-wild models.",
   "red_flags": [
     {
       "flag": "No human baseline",
       "detail": "No human performance data is provided to validate that generated problems preserve difficulty; claims about complexity equivalence rely entirely on the LLM validator and a 30-pair human check."
     },
     {
       "flag": "Contamination study uses only tiny models",
       "detail": "The controlled contamination experiment (Section 4.2) uses only three small models (≤3B parameters); it is unclear if the findings generalize to larger frontier models or instruction-tuned variants."
     },
     {
       "flag": "Human verification sample too small",
       "detail": "Only 30 pairs per dataset (60 total) were reviewed by two graduate students; the 95% agreement rate is based on this small sample with no inter-rater reliability statistics reported."
     },
     {
       "flag": "Generator model dependency",
       "detail": "DyCodeEval relies on CLAUDE-3.5-SONNET for generation; future API changes or model updates would make exact reproduction impossible, and the same model is also evaluated as a benchmark subject (conflict of interest)."
     },
     {
       "flag": "No confirmed code or data release",
       "detail": "Only a project webpage URL is mentioned; no explicit code repository, dataset download, or API access is stated, making reproducibility of reported numbers unclear."
     },
     {
       "flag": "Python-only scope not declared",
       "detail": "All evaluation uses Python-only benchmarks (HumanEval: 164 problems, MBPP: 427 problems), but claims are made about 'Code LLMs' generally without explicitly bounding the scope."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "relevance": "Primary seed dataset; foundational Python code generation benchmark used as input to DyCodeEval"
     },
     {
       "title": "Program Synthesis with Large Language Models (MBPP)",
       "relevance": "Second seed dataset; crowdsourced Python programming benchmark used for evaluation"
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "relevance": "Direct prior work on contamination-free benchmarking via temporal splits from online platforms; key baseline comparison"
     },
     {
       "title": "PPM: Automated Generation of Diverse Programming Problems for Benchmarking Code Generation Models",
       "relevance": "Direct baseline using manually-defined operators to generate problem variants; DyCodeEval outperforms on diversity metrics"
     },
     {
       "title": "DyVal: Dynamic Evaluation of Large Language Models for Reasoning Tasks",
       "relevance": "Related dynamic benchmarking approach using DAG structures; conceptual predecessor to DyCodeEval's dynamic evaluation framework"
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation (EvalPlus)",
       "relevance": "Addresses HumanEval's limited test case coverage; motivates need for rigorous benchmark design"
     },
     {
       "title": "Recent Advances in Large Language Model Benchmarks Against Data Contamination: From Static to Dynamic Evaluation",
       "relevance": "Survey of contamination-free benchmarking landscape that directly contextualizes DyCodeEval's contribution"
     },
     {
       "title": "Generalization or Memorization: Data Contamination and Trustworthy Evaluation for Large Language Models",
       "relevance": "Evidence that contamination is already present in current LLMs; motivates DyCodeEval's approach"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "DyCodeEval provides a usable framework for contamination-resistant benchmarking, though setup requires an LLM agent pipeline."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Data contamination in benchmarks is a known concern; the specific finding about QWEN2.5-CODER-7B adds a mildly surprising result."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Raises concerns about unreliable LLM evaluations due to contamination but does not demonstrate novel attacks or safety threats."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Publicly flagging QWEN2.5-CODER-7B as potentially contaminated and arguing that 'static benchmarks create a false sense of accuracy' has moderate controversy potential."
     },
     "demo_ability": {
       "score": 1,
       "justification": "A project webpage exists but no readily installable tool or live demo is provided in the paper."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Columbia University authors, published at ICML. Evaluates well-known models but is not from a major AI lab."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45537698",
         "title": "Virtual Memory for Real-time RISC-V systems using hPMP",
         "points": 22,
         "comments": 4,
         "url": "https://news.ycombinator.com/item?id=45537698"
       },
       {
         "hn_id": "45115249",
         "title": "When Do Consumers Lose from Variable Electricity Pricing?",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45115249"
       },
       {
         "hn_id": "42966672",
         "title": "Develop AI Agents for System Engineering in Factorio",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42966672"
       },
       {
         "hn_id": "26591284",
         "title": "The Unreasonable Ineffectiveness of Mathematics in Biology",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=26591284"
       },
       {
         "hn_id": "45165536",
         "title": "How to Hack Transformers: Steering LLMs via Prompts, States, and Weight Edits",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=45165536"
       },
       {
         "hn_id": "44798220",
         "title": "An Efficient End-to-End Dynamic Activation Framework for On-Device DNN Training",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44798220"
       },
       {
         "hn_id": "44030713",
         "title": "Cosmos: Predictable and Cost-Effective Adaptation of LLMs",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44030713"
       }
     ],
     "top_points": 22,
     "total_points": 35,
     "total_comments": 5
   }
 }