ai-research-survey

scan-v5.json (20341B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "Dynamic Benchmarking of Reasoning Capabilities in Code Large Language Models Under Data Contamination",
     "authors": [
       "Simin Chen",
       "Pranav Pusarla",
       "Baishakhi Ray"
     ],
     "year": 2025,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2503.04149",
     "doi": "10.48550/arXiv.2503.04149"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All major abstract claims are supported: contamination detection via controlled fine-tuning experiments (§4.2), semantic diversity via BLEU-4 and cosine similarity metrics (§4.4), and stable results across 10 runs (§4.5).",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The controlled contamination experiment (§4.2) intentionally fine-tunes models on leaked benchmark data at 0–100% rates, providing a reasonably controlled basis for the causal claim that DyCodeEval is resistant to contamination-inflated scores.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates only Python NL-to-code generation on two datasets (HumanEval, MBPP) but makes unqualified claims about 'Code LLMs' broadly; no scope boundary is stated for other languages, code tasks like completion or repair, or non-English prompts.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper briefly hypothesizes about the DeepSeek-Coder anomaly but does not discuss whether performance degradation on DyCodeEval for 'potentially contaminated' models could be explained by domain shift from the scenario rewriting or prompt sensitivity rather than contamination.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly proposes DyPass as a better proxy for reasoning capability versus memorization (§5), distinguishing what Pass@K measures from what DyPass@K measures under contamination conditions.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "Limitations are embedded in the conclusion (Section 6) — computational cost and verbose output — but there is no dedicated limitations or threats-to-validity section separate from the conclusion.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions only operational limitations (cost, verbose output) but does not discuss validity threats such as whether fine-tuning contamination simulation reflects real pretraining contamination, or whether the two seed datasets are representative.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated: the paper does not clarify that results are restricted to Python NL-to-code tasks, or that DyCodeEval may not generalize to multilingual settings, code completion, or repair tasks.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the acknowledgements: NSF grants CCF 2313055, CCF 2107405, CAREER 2025082, and FAI: 2040961.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are affiliated with Columbia University's Department of Computer Science, disclosed in the paper header.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders appear to be NSF/government grants independent of the evaluation outcome; no commercial funder with a financial stake in the benchmark results is disclosed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests (patents, equity, consulting) is present in the paper; only NSF grants are acknowledged.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": false,
         "justification": "The central term 'reasoning capability' — which DyCodeEval claims to measure — is never formally defined; the paper conflates reasoning with non-memorization without operationalizing what distinguishes the two.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly lists three contributions: novel problem characterization of static benchmark limitations, the DyCodeEval 4-agent methodology, and empirical findings on contamination resistance and diversity.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 systematically reviews three lines of contamination-free benchmarking and explicitly positions DyCodeEval against PPM (manual effort) and LiveCodeBench (semantic imbalance), showing how it addresses their specific limitations.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper grounds construct validity in metamorphic testing principles: modifying complexity-unrelated context preserves canonical solutions and algorithmic complexity, so the benchmark measures the same underlying capability — a principled if not deeply formalized argument.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "The paper claims complexity-equivalence between seed and generated problems but does not empirically characterize or verify the difficulty distribution (easy/medium/hard tiers) of the generated benchmark items.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "No explicit analysis of ceiling or floor effects is conducted; some models in Fig. 5 score very high on static benchmarks, but the paper does not investigate whether the dynamic benchmark resolves or merely shifts these effects.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline for solving benchmark problems is provided; the human verification step (Appendix D) checks consistency of generated problems but does not measure human solve rates.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Pass@K is formally defined (Equation 1) and the new DyPass@K metric is introduced and explicitly justified as expanding the input space beyond Pass@K to better distinguish reasoning from memorization under contamination (§5).",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "Contamination resistance is the core design principle: a 50×50 scenario-context space, dynamic randomness, and formal collision probability bounds (Theorems 3.1–3.3) collectively make identical problem regeneration extremely unlikely.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether DyCodeEval will remain useful as models improve or as the scenario pool becomes known; no plan for updating scenarios, expanding coverage, or managing benchmark drift over time is provided.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Only superficial failure modes are mentioned (computational cost, verbose prompts); deeper failure modes such as models learning to perform context-stripping, scenario pool exhaustion, or LLM-generated problems sharing statistical patterns are not discussed.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": false,
           "justification": "A project webpage is referenced in the abstract but no explicit code release or reproduction package is described in the paper; full reproduction requires API access to Claude-3.5-Sonnet and reimplementing the generation pipeline from the appendix prompts.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": false,
           "justification": "The seed datasets are described in Appendix B and generation prompts in Appendix C, but the generated dataset itself has no data card, and the scenario pool collection methodology is only partially described.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "No licensing or access terms for DyCodeEval or its generated datasets are stated; the project webpage is referenced but terms of use are not specified in the paper.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": false,
           "justification": "The Impact Statement discusses societal benefits but does not specify what should NOT be concluded from DyCodeEval results, such as inapplicability to non-Python tasks or non-NL-to-code settings.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Static benchmarks create inflated Pass@1 scores for contaminated models, misrepresenting true reasoning capability",
       "evidence": "Controlled fine-tuning with 0–100% leaked data shows steadily increasing Pass@1 on contaminated benchmarks while performance on other benchmarks remains stable (Fig. 4)",
       "supported": "strong"
     },
     {
       "claim": "DyCodeEval prevents contaminated models from achieving artificially inflated benchmark scores",
       "evidence": "In §4.2, models fine-tuned on leaked data show minimal or no improvement on DyCodeEval-generated problems compared to uncontaminated baselines, unlike on static benchmarks",
       "supported": "strong"
     },
     {
       "claim": "QWEN2.5-CODER-7B is potentially contaminated on both HumanEval and MBPP",
       "evidence": "In Fig. 5, QWEN2.5-CODER-7B consistently falls outside the 95% CI of the in-the-wild regression area for both seed datasets",
       "supported": "moderate"
     },
     {
       "claim": "DyCodeEval generates semantically diverse problems while maintaining stable benchmarking results across runs",
       "evidence": "Table 1 shows low BLEU-4 (0.27/0.18) and cosine similarity (0.74/0.73) versus baselines; Fig. 6 shows low standard deviation across 10 independent runs",
       "supported": "strong"
     },
     {
       "claim": "DyPass@K better detects contamination than Pass@K by exposing memorization",
       "evidence": "Table 2: contaminated models show Pass@K inflated to 0.82–0.89 while DyPass@K stays at 0.13–0.17, versus uncontaminated models where both metrics align",
       "supported": "strong"
     },
     {
       "claim": "The probability of generating identical problems across DyCodeEval runs is negligibly low",
       "evidence": "Theorems 3.1–3.3 provide formal probability bounds under uniform sampling from a 50×50 scenario-context space with mathematical proofs in Appendix A",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "DyCodeEval is a 4-agent pipeline (Scenario Proposer, Context Generator, Prompt Rewriter, Validator) that generates semantically diverse but algorithmically equivalent programming problems by modifying complexity-unrelated context while preserving canonical solutions, using metamorphic testing principles to resist data contamination. Controlled contamination experiments show that static benchmarks inflate Pass@1 for models fine-tuned on leaked data, while DyCodeEval remains resistant. In-the-wild evaluation across 12+ models identifies QWEN2.5-CODER-7B as a regression outlier suggesting potential contamination on both HumanEval and MBPP. The new DyPass metric more faithfully measures reasoning capability under contamination than Pass@K by varying the prompt context rather than sampling multiple solutions.",
   "red_flags": [
     {
       "flag": "Contamination simulation mismatch",
       "detail": "The paper simulates contamination via fine-tuning on leaked benchmark data, but real pretraining contamination occurs during large-scale training; models may respond very differently to these two types, and this validity threat is not discussed."
     },
     {
       "flag": "Reasoning undefined",
       "detail": "The central claim — that DyCodeEval measures 'reasoning capability' rather than memorization — relies on an undefined distinction; 'reasoning' is never operationalized, leaving the core theoretical contribution unverifiable."
     },
     {
       "flag": "Scope overclaim",
       "detail": "Evaluated only on Python NL-to-code on two benchmarks, but conclusions are framed as applying to 'Code LLMs' broadly without bounding results to this specific task type and language."
     },
     {
       "flag": "No human baseline",
       "detail": "No human performance baseline on benchmark problems is provided; it is unknown whether scenario-rewritten variants introduce unintended difficulty beyond what the metamorphic testing argument predicts."
     },
     {
       "flag": "QWEN contamination claim is weak",
       "detail": "Labeling QWEN2.5-CODER-7B as 'potentially contaminated' based solely on regression outlier status is a weak statistical claim; alternative explanations such as training methodology differences are not considered."
     },
     {
       "flag": "Reproducibility barrier",
       "detail": "Full reproduction requires API access to Claude-3.5-Sonnet for problem generation; no explicit code release is described in the paper, and the generated datasets are not shared as a static artifact."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "relevance": "Foundational code generation benchmark used as the primary seed dataset for DyCodeEval"
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "relevance": "Key prior work on contamination-aware temporal benchmarking that DyCodeEval explicitly improves upon"
     },
     {
       "title": "PPM: Automated Generation of Diverse Programming Problems for Benchmarking Code Generation Models",
       "relevance": "Direct predecessor that DyCodeEval addresses, overcoming its manual operator definition and semantic imbalance limitations"
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of LLMs for Code Generation (EvalPlus)",
       "relevance": "Prior work identifying HumanEval's limited test coverage, relevant to the benchmark rigor discussion"
     },
     {
       "title": "DyVal: Dynamic Evaluation of Large Language Models for Reasoning Tasks",
       "relevance": "Prior dynamic evaluation approach using DAG structures — related methodology and conceptual predecessor"
     },
     {
       "title": "ReCode: Robustness Evaluation of Code Generation Models",
       "relevance": "Used as comparison baseline for mutation-based diversity methods in Table 1"
     },
     {
       "title": "Program Synthesis with Large Language Models (MBPP)",
       "relevance": "Second seed dataset used across all DyCodeEval experiments"
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "Representative code LLM benchmark cited in the broader benchmarking landscape survey"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly addresses data contamination in LLM evaluation — a critical problem for any team benchmarking or comparing code LLMs in practice."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Provides suggestive evidence that QWEN2.5-CODER-7B may be contaminated on standard benchmarks, implicitly challenging published benchmark results."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Contaminated benchmarks give false confidence in AI capabilities, with mild but real implications for deployment decisions — not primarily a safety paper."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Implicitly accuses QWEN2.5-CODER-7B of potential benchmark contamination, which is a commercially and reputationally charged claim against a specific named model."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Project webpage is provided and all prompts are fully documented in the appendix; practitioners could reproduce the pipeline with API access to Claude-3.5-Sonnet."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Columbia University is a reputable institution but not a top AI lab; the paper uses Claude models but is not authored by Anthropic or Google or Meta."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45537698",
         "title": "Virtual Memory for Real-time RISC-V systems using hPMP",
         "points": 22,
         "comments": 4,
         "url": "https://news.ycombinator.com/item?id=45537698"
       },
       {
         "hn_id": "45115249",
         "title": "When Do Consumers Lose from Variable Electricity Pricing?",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45115249"
       },
       {
         "hn_id": "42966672",
         "title": "Develop AI Agents for System Engineering in Factorio",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42966672"
       },
       {
         "hn_id": "26591284",
         "title": "The Unreasonable Ineffectiveness of Mathematics in Biology",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=26591284"
       },
       {
         "hn_id": "45165536",
         "title": "How to Hack Transformers: Steering LLMs via Prompts, States, and Weight Edits",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=45165536"
       },
       {
         "hn_id": "44798220",
         "title": "An Efficient End-to-End Dynamic Activation Framework for On-Device DNN Training",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44798220"
       },
       {
         "hn_id": "44030713",
         "title": "Cosmos: Predictable and Cost-Effective Adaptation of LLMs",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44030713"
       }
     ],
     "top_points": 22,
     "total_points": 35,
     "total_comments": 5
   }
 }