ai-research-survey

scan-v5.json (21937B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "DynaCode: A Dynamic Complexity-Aware Code Benchmark for Evaluating Large Language Models in Code Generation",
     "authors": [
       "Wenhao Hu",
       "Jinhao Duan",
       "Chunchen Wei",
       "Li Zhang",
       "Yue Zhang",
       "Kaidi Xu"
     ],
     "year": 2025,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2503.10452",
     "doi": "10.48550/arXiv.2503.10452"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's core claims — 189M unique problems, 16.8–45.7% average performance drop vs MBPP+, and contamination resistance — are all verified by Table 1 (combinatorial problem counts), Table 2 (performance results per model), and the fine-tuning experiment in Figure 5.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims DynaCode 'limits memorization' and that performance drops are caused by complexity, but the fine-tuning experiment cannot separate 'harder to memorize' from 'harder in general' — both predict smaller gains on DynaCode after fine-tuning. No control isolates these confounds.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'LLMs are good at sequential execution' and 'LLMs struggle with problem understanding as complexity increases' are stated as general conclusions about LLMs, but the error analysis covers only GPT-3.5-Turbo and the call-graph analysis covers only 4 of 12 models.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Performance drops with complexity could be attributable to longer prompt length, larger number of required functions, or type-alignment constraints in call-graph construction — none of these alternative explanations are discussed.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Pass@1 measures whether generated code passes execution-based test cases; the paper consistently uses this to claim evaluation of 'code generation capability,' which is a reasonable direct measurement rather than a proxy.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated Limitations section appears at the end of the paper, discussing maximum node count of 5 in call graphs and plans for future extension to more complex structures.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section only notes 'maximum node count of 5' without discussing validity threats such as: cyclomatic complexity as a difficulty proxy, quality of base problems from MBPP+/LeetCode, Python-only scope, or type-alignment sampling bias. Too thin to count as specific threats-to-validity.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state scope boundaries — e.g., that results apply only to Python code generation, only to function-level tasks, or that findings should not be generalized to repository-level or multi-file generation tasks.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgment or funding section appears anywhere in the paper. Absence of any funding disclosure.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors' affiliations are clearly stated: University of Electronic Science and Technology of China (Hu, Wei) and Drexel University (Duan, L. Zhang, Y. Zhang, Xu).",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so this criterion does not apply.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, no patent or equity declarations appear anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are formally defined: cyclomatic complexity with equation (1), call graph as a directed acyclic graph with specified properties, 'unit' and 'level' with explicit threshold formulas (equations 2–4), and 'complexity matrix' (equation 5).",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions are enumerated in a bullet list at the end of the introduction: a dynamic evaluation strategy, complexity-aware metrics combining code and graph complexity, and the DynaCode benchmark with multi-LLM evaluation.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 covers related work on dynamic evaluation (DyVal, NPHardEval, DARG, Benchmark Self-Evolving) and coding benchmarks (HumanEval, MBPP, BigCodeBench, SWE-Bench), explicitly positioning DynaCode against each and explaining its differences.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues that cyclomatic complexity captures control-flow branching that LLMs demonstrably struggle with (citing Jiang et al., 2025; Beger and Dutta, 2025) and that call-graph structures test inter-function dependency handling — both are linked to identified LLM failure modes.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 shows problem distribution across 4 complexity units and 4 graph levels. Figure 10 validates that Pass@1 scores degrade systematically with both unit and level complexity across all 12 models, and Figure 8 shows examples of increasing problem complexity.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not explicitly check for ceiling or floor effects. codegemma-7b-it scores 2.9% on DynaCode (near floor), and several weak models drop below 10% on harder units — potential floor effects for weak models are not discussed.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human performance baseline is provided anywhere in the paper. There is no discussion of how a human programmer would perform on DynaCode problems at any complexity level.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Pass@1 is justified by reference to prior work (Chen et al., 2021) and its use enables consistent comparison with MBPP and MBPP+. Pass@3 results are also reported in Appendix D for robustness validation.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "Contamination resistance is a central design goal: 189M unique combinations make memorization impractical, new LeetCode problems are actively sourced for continuous refresh, and the fine-tuning ablation (Figure 5) empirically tests and supports contamination resistance.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The Limitations section acknowledges that advanced LLMs may eventually handle current call-graph structures and mentions plans to extend to more complex structures. The dynamic generation mechanism (continually adding new base problems from the web) provides inherent temporal adaptability.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss failure modes of the benchmark itself — e.g., type-alignment constraints that may bias which problems can be combined, Monkeytype annotation quality issues, or cases where dynamically combined prompts become semantically incoherent.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Benchmark and evaluation code are publicly available at https://github.com/HWH-2000/DynaCode, as stated in the abstract and confirmed by the detailed appendix examples.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Section 3 and appendices document sources (MBPP+, LeetCode), preprocessing pipeline (Monkeytype type annotation, bad-generation filtering), benchmark statistics (Tables 1, 5, 6), example prompts (Table 8), and all 16 call-graph structures (Figure 9).",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "No licensing information is provided in the paper. The use of LeetCode problems raises unaddressed copyright concerns. Only a GitHub link is provided with no stated license terms for the benchmark or its components.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper states DynaCode is for 'evaluating LLMs on code generation tasks' but does not specify what should NOT be concluded — e.g., that high DynaCode scores don't imply real-world coding capability or multi-file task competence.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "DynaCode generates up to 189 million unique nested code problems across 4 complexity units and 16 call-graph structures",
       "evidence": "Table 1 shows combinatorial counts per unit/level; Table 5 in the appendix provides per-graph-type breakdowns summing to 189,263,141 total problems",
       "supported": "strong"
     },
     {
       "claim": "Evaluated LLMs show an average performance drop of 16.8% to 45.7% on DynaCode compared to MBPP+",
       "evidence": "Table 2: GPT-4o drops from 72.2% (MBPP+) to 55.4% (DynaCode) = 16.8pp; Meta-Llama-3.1-8B drops from 55.6% to 9.9% = 45.7pp, matching the stated range exactly",
       "supported": "strong"
     },
     {
       "claim": "DynaCode resists memorization: fine-tuned models show large gains on MBPP+ but minimal gains on DynaCode",
       "evidence": "Figure 5: GPT-3.5-Turbo fine-tuned on MBPP+ jumps 69.7%→88.6% on MBPP+ but only 32.6%→36.0% on DynaCode; Meta-Llama-3.1-8B jumps 55.6%→98.1% on MBPP+ but only 10.6%→23.6% on DynaCode",
       "supported": "moderate"
     },
     {
       "claim": "LLMs perform significantly better on sequential call graphs (G1–G4, G8) than multi-branch complex call graphs (G9–G16)",
       "evidence": "Figure 6 shows consistent higher Pass@1 on sequential graphs vs. complex branches across all 4 tested models (GPT-4o, GPT-3.5-Turbo, WizardLM-2-8x22B, Meta-Llama-3.1-405B)",
       "supported": "strong"
     },
     {
       "claim": "Problem Understanding error rate increases from 64.1% (Unit 1) to 88.8% (Unit 4) as complexity rises",
       "evidence": "Table 3 reports error categorization for GPT-3.5-Turbo only, based on 100 questions per call graph; generalizing to 'LLMs' is unsupported by this single-model analysis",
       "supported": "moderate"
     },
     {
       "claim": "Models with known data contamination (Meta-Llama-3-8B-Instruct) show disproportionately large performance drops on DynaCode",
       "evidence": "Figure 1 shows Meta-Llama-3-8B drops from 64.6% (MBPP) to 8.4% (DynaCode) while DeepSeek-V3 drops from 87.6% to 52.1%; only 2 models shown, contamination status asserted not proven",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "DynaCode generates 189M unique Python code generation problems by combining base problems from MBPP+ and LeetCode via directed acyclic call-graph structures, classifying problems by cyclomatic complexity (4 units) and graph topology (4 levels). Evaluated on 12 LLMs, all models show systematic performance degradation as both code and graph complexity increase, with average drops of 16.8–45.7pp versus MBPP+. Fine-tuning experiments provide evidence that DynaCode is more resistant to memorization than static benchmarks, as models that achieve near-perfect scores on MBPP+ after fine-tuning show only marginal improvements on DynaCode. Error analysis on GPT-3.5-Turbo reveals that Problem Understanding failures (not syntax or context errors) become dominant at higher complexity levels, growing from 64.1% to 88.8% of errors across units.",
   "red_flags": [
     {
       "flag": "No human baseline",
       "detail": "No human programmer performance is reported at any complexity level, making it impossible to assess whether DynaCode's harder units discriminate meaningful capability differences or represent poorly constructed tasks beyond reasonable human ability."
     },
     {
       "flag": "Causal contamination claim confounded",
       "detail": "The fine-tuning experiment cannot separate 'DynaCode resists memorization' from 'DynaCode is harder regardless of memorization' — both hypotheses predict identical patterns of smaller gains after fine-tuning on DynaCode vs. MBPP+."
     },
     {
       "flag": "Error analysis on single model only",
       "detail": "Table 3's error distribution analysis covers only GPT-3.5-Turbo; generalizing conclusions about error types across 'LLMs' broadly from one model is unsupported."
     },
     {
       "flag": "LeetCode licensing unaddressed",
       "detail": "The benchmark incorporates LeetCode problems and their official test cases without any discussion of copyright, licensing, or terms of use — this could limit legal reproducibility and distribution of the dataset."
     },
     {
       "flag": "Python-only scope never stated",
       "detail": "All benchmark problems are in Python, but this critical scope limitation is never explicitly called out; conclusions about LLM code generation capabilities cannot be assumed to transfer to other programming languages."
     },
     {
       "flag": "Type-alignment constraint biases problem selection",
       "detail": "Call-graph construction requires output types of parent functions to match input types of children — this constraint silently filters the combinatorial space and may systematically favor functions with common return types, biasing difficulty distribution."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "relevance": "Foundational code generation benchmark and source of Pass@k metric adopted in DynaCode; primary baseline for comparison"
     },
     {
       "title": "Program Synthesis with Large Language Models (MBPP)",
       "relevance": "Primary source dataset for DynaCode unit functions; main static benchmark comparison throughout"
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation (EvalPlus/MBPP+)",
       "relevance": "MBPP+ (EvalPlus-processed) is the direct unit function source for DynaCode; provides enhanced test cases"
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "relevance": "Contemporary benchmark representing state-of-the-art that DynaCode is positioned against in Table 6"
     },
     {
       "title": "DyVal: Graph-Informed Dynamic Evaluation of Large Language Models",
       "relevance": "Prior graph-based dynamic evaluation work; DynaCode extends this approach to the code generation domain"
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "relevance": "Alternative contamination-resistant benchmark that uses temporal splits rather than dynamic generation"
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "Representative real-world code benchmark discussed as complementary, targeting repository-level tasks vs. DynaCode's function-level focus"
     },
     {
       "title": "DyVal 2: Dynamic Evaluation of Large Language Models by Meta Probing Agents",
       "relevance": "Related LLM-agent-driven dynamic evaluation approach that DynaCode explicitly distinguishes from by avoiding LLM-as-evaluator instability"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners evaluating LLMs for code generation tasks can use DynaCode to stress-test models under controlled complexity, though the Python-only scope and pipeline setup overhead limit immediate broad applicability."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The magnitude of performance drops (up to 45.7pp below MBPP+ scores) is striking and challenges confidence in existing benchmark scores as reliable indicators of real-world coding capability."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or risk concerns are raised; the paper is purely about benchmark methodology and evaluation."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicit challenge to the validity of widely-used static benchmarks (HumanEval, MBPP) as reliable evaluation tools, framed constructively rather than controversially."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is publicly available on GitHub and the dynamic generation pipeline allows others to generate benchmarks and run evaluations immediately on their own models."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from Drexel University and UESTC — credible academic institutions but no major AI lab (OpenAI, Google, Meta, DeepMind) affiliation to drive recognition."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "44592304",
         "title": "Mixture-of-Recursions: Learning Adaptive Token-Level Computation",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44592304"
       },
       {
         "hn_id": "43288456",
         "title": "Computation-Aware ControlNet with Dynamic Router for Text-to-Image Generation",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43288456"
       },
       {
         "hn_id": "45328070",
         "title": "Why Johnny Cant Use Agents: Aspirations vs. Realities with AI Agents",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45328070"
       },
       {
         "hn_id": "35263649",
         "title": "A comprehensive capacity analysis of GPT-3 and GPT-3.5 models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=35263649"
       },
       {
         "hn_id": "26536350",
         "title": "Dynamic Kernel Matching for Non-Conforming Data: A Study of T-Cell Receptors",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=26536350"
       },
       {
         "hn_id": "45467729",
         "title": "AegisShield: Democratizing Cyber Threat Modeling with Generative AI",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45467729"
       },
       {
         "hn_id": "44634645",
         "title": "Mixture-of-Recursions: Learning Dynamic Recursive Depths",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44634645"
       },
       {
         "hn_id": "44579442",
         "title": "Mixture-of-Recursions",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44579442"
       },
       {
         "hn_id": "44008034",
         "title": "Emotion-Sensitive Explanation Model",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44008034"
       },
       {
         "hn_id": "43349900",
         "title": "FlexControl: Dynamic Block Activation for Diffusion Models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43349900"
       }
     ],
     "top_points": 3,
     "total_points": 17,
     "total_comments": 0
   }
 }