ai-research-survey

scan-v4.json (21285B)

---

 {
   "scan_version": 4,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "DynaCode: A Dynamic Complexity-Aware Code Benchmark for Evaluating Large Language Models in Code Generation",
     "authors": [
       "Wenhao Hu",
       "Jinhao Duan",
       "Chunchen Wei",
       "Li Zhang",
       "Yue Zhang",
       "Kaidi Xu"
     ],
     "year": 2025,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2503.10452",
     "doi": "10.48550/arXiv.2503.10452"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'average performance drop of 16.8% to 45.7% compared to MBPP+' — Table 2 supports this across models. '189 million unique nested code problems' is supported by Table 1 (total 189,263,141). 'Performance progressively decreasing as complexity increases' is shown in Figure 3 and Table 2.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'DynaCode's dynamic evaluation strategy effectively mitigates data contamination' and 'LLMs struggle with parallel function dependencies.' These are causal claims. The fine-tuning experiment provides some evidence for the contamination claim but has confounds (different training set sizes, different data distributions). The complexity claims don't control for prompt length, which increases with call-graph complexity.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Evaluating Large Language Models in Code Generation' broadly, but the benchmark tests only Python, only function-level composition from MBPP+, with a maximum of 5 nodes in call graphs. The paper does not bound its claims to these constraints. The Limitations section mentions the 5-node limit but does not address the language or task-type restrictions.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed performance drops. For example, longer prompts from nested problems could independently cause degradation. Performance differences across call-graph types could be due to prompt structure rather than genuine code understanding differences. No confounds are discussed.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Pass@1 on code generation tasks and frames results in terms of Pass@1 scores and code generation capability. The claims largely match the granularity of the measurements — they don't inflate Pass@1 into broader 'programming ability' claims.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after the Conclusion, discussing the maximum node count of 5 in call graphs and future extensions to more complex structures.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section only mentions the 5-node call-graph constraint. It does not discuss specific threats like prompt-length confounds, reliance on MBPP+ as the sole base problem source, or potential biases in cyclomatic complexity as the sole complexity metric.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It doesn't note that results are limited to Python, to function-level composition, to a specific set of base problems, or that the complexity metric captures only one dimension of real-world code difficulty.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is mentioned anywhere in the paper.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Electronic Science and Technology of China and Drexel University. The authors are not affiliated with any of the model vendors being evaluated.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be verified. The authors are academic researchers not affiliated with model vendors, but without a funding disclosure, this criterion is not satisfied.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest declaration is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Cyclomatic complexity is defined with the formal formula (Eq. 1), call graphs are formally defined as directed acyclic graphs with specified properties, complexity units and levels are defined via threshold intervals (Eq. 2, 4), and the call-graph complexity metric M is explicitly stated (Eq. 3).",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly lists three contributions: the dynamic evaluation strategy, the complexity-aware metric combining code and call-graph complexity, and the DynaCode benchmark itself with multi-LLM evaluation.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 covers both dynamic evaluation methods (DyVal, NPHardEval, DyVal2, DARG) and static code benchmarks (HumanEval, MBPP, EvalPlus, BigCodeBench, SWE-Bench), explicitly contrasting DynaCode's approach against each category.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues that cyclomatic complexity is chosen specifically because 'LLMs often struggle with code generation tasks involving complex control flow and branching logic,' and that call-graph nesting captures inter-function dependencies absent from isolated-function benchmarks.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 reports problem counts across 4 complexity units and 4 graph levels; Table 2 and Figure 3 empirically confirm monotonic performance degradation across units, validating that the difficulty tiers separate as intended.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "codegemma-7b-it averages 2.9% on DynaCode (near floor) and the paper does not flag or discuss this floor effect; no ceiling analysis is performed for top models.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline performance is reported for any DynaCode problem set, leaving it unclear whether benchmark items are solvable by competent programmers and what a ceiling should look like.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": false,
           "justification": "Pass@1 is adopted with the sole justification of 'following previous work (Chen et al., 2021)'; no argument is made for why Pass@1 is appropriate for multi-function nested code or how partial credit for individual subfunctions should be handled.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "Dynamic generation of ~189M combinations and the demonstrated fine-tuning ablation (Figure 5) directly show that memorizing unit functions provides little benefit on nested variants, constituting an explicit contamination-resistance mechanism.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The paper notes active sourcing of new LeetCode problems to refresh the unit pool and explicitly states in the Limitations section that call-graph structures will be extended as models improve, constituting a rudimentary update plan.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The only failure mode acknowledged is the 5-node cap on call graphs; the paper does not discuss failure modes such as models gaming call-graph patterns, prompt length confounds, or the benchmark's unsuitability for multi-file or non-Python tasks.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "The abstract and Section 5 state that 'our benchmark and evaluation code are available at https://github.com/HWH-2000/DynaCode,' enabling reproduction of reported numbers.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "The paper documents source data (MBPP+, LeetCode), construction methodology (complexity classification, call-graph construction, test-case generation), filtering steps (bad generation removal), and detailed statistics in Tables 1 and 5.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "A GitHub URL is provided but no license for DynaCode itself is stated; the LeetCode problems incorporated have restrictive terms of service that the paper does not address, leaving the legal status of redistribution unclear.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not state what DynaCode is NOT appropriate for (e.g., evaluating multi-file completion, non-Python languages, agents), nor does it warn against over-interpreting results from models already trained on MBPP+.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "DynaCode can generate up to 189 million unique nested code problems across 4 complexity units and 16 call-graph structures.",
       "evidence": "Table 1 and Table 5 provide detailed combinatorial counts summing to 189,263,141 problems.",
       "supported": "strong"
     },
     {
       "claim": "All 12 evaluated LLMs show average performance drops of 16.8% to 45.7% on DynaCode compared to MBPP+.",
       "evidence": "Table 2 reports Pass@1 scores for all 12 models on both MBPP+ and DynaCode; the ranges are directly computable from the data.",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning on MBPP+ causes large gains on MBPP+ but minimal gains on DynaCode, demonstrating DynaCode's contamination resistance.",
       "evidence": "Figure 5 shows GPT-3.5-Turbo improves from 69.7% to 88.6% on MBPP+ but only 32.6% to 36.0% on DynaCode after fine-tuning.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs perform significantly better on sequential call graphs (G1–G4, G8) than on multi-branch structures (G9–G16).",
       "evidence": "Figure 6 shows this pattern for 4 evaluated models, though the analysis does not cover all 12 benchmark participants.",
       "supported": "moderate"
     },
     {
       "claim": "Problem Understanding errors in GPT-3.5-Turbo increase monotonically from 64.1% in Unit 1 to 88.8% in Unit 4 as complexity rises.",
       "evidence": "Table 3 reports error distribution across 4 units for GPT-3.5-Turbo; findings generalized to 'LLMs' in section heading despite single-model scope.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "DynaCode dynamically generates ~189M nested Python code problems by combining existing unit functions via call-graph structures with 4 cyclomatic complexity tiers and 16 graph topologies, achieving massive diversity that resists memorization. Evaluation of 12 LLMs shows consistent performance degradation as complexity increases, with average drops of 16.8–45.7% versus the static MBPP+ benchmark. A controlled fine-tuning experiment demonstrates that memorizing unit functions provides minimal benefit on full DynaCode problems, empirically validating contamination resistance. LLMs systematically favor sequential call graphs over multi-branch structures, with error analysis on GPT-3.5-Turbo showing problem-understanding failures dominating at higher complexity levels.",
   "red_flags": [
     {
       "flag": "No human baseline",
       "detail": "No human performance data is provided for any complexity level; it is unknown whether the hardest problems are solvable by competent programmers or whether low model scores reflect task validity issues."
     },
     {
       "flag": "Floor effects uninvestigated",
       "detail": "codegemma-7b-it averages 2.9% on DynaCode and several models approach floor-level performance, but the paper does not flag or analyze floor effects."
     },
     {
       "flag": "Error analysis single-model overgeneralization",
       "detail": "Table 3 covers only GPT-3.5-Turbo, but Section 4.3 titles are written as universal claims about 'LLMs' without qualifying that findings apply to one model."
     },
     {
       "flag": "Call-graph complexity metric unjustified",
       "detail": "The metric M = Lmax × B × |E| (product of path length, branch count, and edge count) is proposed without theoretical or empirical justification for why the multiplicative form is appropriate versus, e.g., additive or weighted alternatives."
     },
     {
       "flag": "LeetCode ToS not addressed",
       "detail": "LeetCode problems incorporated into DynaCode have restrictive terms of service; the paper does not address whether these problems can be redistributed as part of the benchmark."
     },
     {
       "flag": "No funding disclosure",
       "detail": "The paper contains no acknowledgments section or funding disclosure despite multiple institutional affiliations."
     },
     {
       "flag": "Thin limitations section",
       "detail": "The Limitations section is a single paragraph mentioning only the 5-node cap on call graphs, leaving major scope limitations (Python-only, isolated functions, single-attempt evaluation) unacknowledged."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code (HumanEval)",
       "relevance": "Foundational code benchmark and Pass@k metric that DynaCode builds upon and compares against"
     },
     {
       "title": "Program synthesis with large language models (MBPP)",
       "relevance": "Primary source of unit functions used in DynaCode; the MBPP+ variant is the main comparison baseline"
     },
     {
       "title": "Is your code generated by ChatGPT really correct? rigorous evaluation of LLMs for code generation (EvalPlus)",
       "relevance": "Produces MBPP+ (the enhanced unit function set) and the augmented test case methodology adopted here"
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "relevance": "Contemporary code benchmark positioned as a point of comparison in Table 6 on problem scale"
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "relevance": "Real-world code benchmark situating DynaCode within the landscape of increasingly complex code evaluation"
     },
     {
       "title": "DyVal: Graph-informed dynamic evaluation of large language models",
       "relevance": "Direct methodological precedent for graph-based dynamic evaluation; DynaCode extends this idea to code generation"
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "relevance": "Another contamination-aware code benchmark; directly relevant for situating DynaCode's anti-contamination claims"
     },
     {
       "title": "EvoEval: Evolving coding benchmarks via LLM",
       "relevance": "Dynamic benchmark creation via LLM transformation; contrasted with DynaCode's automated structural approach"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Released benchmark tool that practitioners can use to evaluate LLMs on code generation, though limited to Python function composition."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms the expected finding that LLMs struggle with increased complexity; the magnitude of performance drops (up to 45.7pp) is noteworthy but not fundamentally surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; purely a benchmark evaluation paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Shows popular benchmarks like MBPP are contaminated and unreliable, but data contamination in LLM benchmarks is already well-established."
     },
     "demo_ability": {
       "score": 2,
       "justification": "GitHub repository released with benchmark generation and evaluation code; users could run it on their own models."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates well-known models (GPT-4o, DeepSeek-V3, Llama-3) but paper is from UESTC/Drexel, not a major AI lab."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "44592304",
         "title": "Mixture-of-Recursions: Learning Adaptive Token-Level Computation",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44592304"
       },
       {
         "hn_id": "43288456",
         "title": "Computation-Aware ControlNet with Dynamic Router for Text-to-Image Generation",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43288456"
       },
       {
         "hn_id": "45328070",
         "title": "Why Johnny Cant Use Agents: Aspirations vs. Realities with AI Agents",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45328070"
       },
       {
         "hn_id": "35263649",
         "title": "A comprehensive capacity analysis of GPT-3 and GPT-3.5 models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=35263649"
       },
       {
         "hn_id": "26536350",
         "title": "Dynamic Kernel Matching for Non-Conforming Data: A Study of T-Cell Receptors",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=26536350"
       },
       {
         "hn_id": "45467729",
         "title": "AegisShield: Democratizing Cyber Threat Modeling with Generative AI",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45467729"
       },
       {
         "hn_id": "44634645",
         "title": "Mixture-of-Recursions: Learning Dynamic Recursive Depths",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44634645"
       },
       {
         "hn_id": "44579442",
         "title": "Mixture-of-Recursions",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44579442"
       },
       {
         "hn_id": "44008034",
         "title": "Emotion-Sensitive Explanation Model",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44008034"
       },
       {
         "hn_id": "43349900",
         "title": "FlexControl: Dynamic Block Activation for Diffusion Models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43349900"
       }
     ],
     "top_points": 3,
     "total_points": 17,
     "total_comments": 0
   }
 }