ai-research-survey

scan.json (32264B)

---

 {
   "paper": {
     "title": "DynaCode: A Dynamic Complexity-Aware Code Benchmark for Evaluating Large Language Models in Code Generation",
     "authors": [
       "Wenhao Hu",
       "Jinhao Duan",
       "Chunchen Wei",
       "Li Zhang",
       "Yue Zhang",
       "Kaidi Xu"
     ],
     "year": 2025,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2503.10452",
     "doi": "10.48550/arXiv.2503.10452"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our benchmark and evaluation code are available at https://github.com/HWH-2000/DynaCode' — a concrete URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is dynamically generated from MBPP+ (a publicly available dataset via EvalPlus) and LeetCode problems. The generation code is released on GitHub, allowing reconstruction of the full benchmark."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is mentioned in the paper. No library versions are listed beyond the tools used (Radon, Monkeytype)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. A GitHub link is given but the paper does not describe how to run the benchmark or reproduce specific results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports ± values for all DynaCode results, e.g., 'GPT-4o: 74.4 (±1.6)' for Unit 1. These represent standard deviation across runs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'GPT-4o consistently outperforms others') based solely on comparing point estimates. No statistical significance tests (t-test, bootstrap, etc.) are reported anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance drops are reported with full context, e.g., 'GPT-4o achieves 87.6% on MBPP and 72.2% on MBPP+, but drops significantly to 55.4% on DynaCode.' Abstract states 'average performance drop of 16.8% to 45.7%.' Baseline numbers always accompany claims."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 ('Effect of the Problem Sizes') explicitly studies the effect of problem count {25, 50, 75, 100, 125, 150} on evaluation stability (Figure 4) and concludes 'when the number of problems is greater than or equal to 75, the evaluation results meet our requirements for stability.' They chose N=100."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 states 'all experiments were conducted three times with 5 different random seeds, and the average results are presented' with ± standard deviation reported for all DynaCode results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Results are compared against MBPP and MBPP+ benchmarks across all 12 models (Table 2). The fine-tuning experiment (Figure 5) also includes baseline (non-fine-tuned) performance."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "MBPP+ (EvalPlus version) is a contemporary baseline. The evaluated models include recent ones like GPT-4o, DeepSeek-V3, Qwen2.5-Coder, and Meta-Llama-3.3-70B-Instruct."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark design is systematically decomposed: results are broken down by code complexity unit (1-4), call-graph level (1-4), and individual call-graph structure (G1-G16). The fine-tuning experiment ablates by training source (MBPP+ vs DynaCode vs unit functions). The problem size study varies N."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Pass@1 is the primary metric (Table 2) and Pass@3 is reported in Table 4 (Appendix D.1). The schema examples explicitly list 'Pass@1 AND Pass@10' as qualifying for multiple metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via pass/fail test case execution. No human evaluation of generated code quality, readability, or correctness is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For the main evaluation, models are evaluated off-the-shelf (temperature=0) on dynamically generated problems, so no dev/test contamination is possible. The benchmark is designed so test problems are guaranteed unseen."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns: by code complexity unit (Table 2), by call-graph level (Figure 3), by individual call graph G1-G16 (Figure 6), by error type and unit (Table 3), and per-model detailed trends (Figure 10)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 presents detailed error analysis: 4,279 error examples from GPT-3.5-Turbo categorized into Problem Understanding, Code Pattern Generation, and Context Management (Table 3). Error distribution across units is analyzed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Fine-tuning on DynaCode unit functions actually decreases performance (GPT-3.5 drops from 32.6% to 15.2%, Meta-Llama from 10.6% to 6.9% in Figure 5). Models like codegemma-7b-it show near-zero performance (2.9% average). These are reported without downplaying."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'average performance drop of 16.8% to 45.7% compared to MBPP+' — Table 2 supports this across models. '189 million unique nested code problems' is supported by Table 1 (total 189,263,141). 'Performance progressively decreasing as complexity increases' is shown in Figure 3 and Table 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'DynaCode's dynamic evaluation strategy effectively mitigates data contamination' and 'LLMs struggle with parallel function dependencies.' These are causal claims. The fine-tuning experiment provides some evidence for the contamination claim but has confounds (different training set sizes, different data distributions). The complexity claims don't control for prompt length, which increases with call-graph complexity."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Evaluating Large Language Models in Code Generation' broadly, but the benchmark tests only Python, only function-level composition from MBPP+, with a maximum of 5 nodes in call graphs. The paper does not bound its claims to these constraints. The Limitations section mentions the 5-node limit but does not address the language or task-type restrictions."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed performance drops. For example, longer prompts from nested problems could independently cause degradation. Performance differences across call-graph types could be due to prompt structure rather than genuine code understanding differences. No confounds are discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Pass@1 on code generation tasks and frames results in terms of Pass@1 scores and code generation capability. The claims largely match the granularity of the measurements — they don't inflate Pass@1 into broader 'programming ability' claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Open models are well-specified (e.g., 'Meta-Llama-3.1-8B-Instruct', 'Qwen2.5-Coder-32B-Instruct'). However, 'GPT-4o' and 'GPT-3.5-Turbo' lack snapshot dates or API versions. Per the schema, marketing names like 'GPT-4o' without a snapshot date do not count as specified versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E describes the prompt construction process and Table 8 provides a full example prompt for call graph G8. The prompts are dynamically generated from a documented hard-coding strategy, and the actual text is shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states 'we set the temperature to 0 to eliminate any randomness during the generation process.' For fine-tuning experiments, epochs and total steps are reported (e.g., '5 epochs', '1890 steps')."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated directly via single-turn prompting with temperature=0."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 documents the full pipeline: problem collection from MBPP+ and LeetCode, cyclomatic complexity classification using Radon, call-graph construction, type-based problem combination via Monkeytype, test case generation with batch execution, and filtering of bad generations."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after the Conclusion, discussing the maximum node count of 5 in call graphs and future extensions to more complex structures."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section only mentions the 5-node call-graph constraint. It does not discuss specific threats like prompt-length confounds, reliance on MBPP+ as the sole base problem source, or potential biases in cyclomatic complexity as the sole complexity metric."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It doesn't note that results are limited to Python, to function-level composition, to a specific set of base problems, or that the complexity metric captures only one dimension of real-world code difficulty."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark generation code is released, but raw experimental outputs (model-generated code, per-problem pass/fail results) are not mentioned as being available. Only aggregated results are shown in tables and figures."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 and 4.1 describe data sources: MBPP+ from EvalPlus as the primary unit function set, plus 40 new problems from LeetCode (22 for Unit 3, 18 for Unit 4). Collection criteria and integration process are explained."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (MBPP+) and public programming problems (LeetCode)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented across Sections 3.1-3.3: MBPP+ problems → cyclomatic complexity computation via Radon → unit classification → call-graph construction → type-based combination → test case generation → bad generation filtering. Table 1 shows problem counts at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Electronic Science and Technology of China and Drexel University. The authors are not affiliated with any of the model vendors being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be verified. The authors are academic researchers not affiliated with model vendors, but without a funding disclosure, this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses data contamination extensively but does not state the training data cutoff dates for any of the 12 evaluated models. Without cutoff dates, it's impossible to verify which benchmarks were in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper's motivation is addressing train/test overlap. Section 1 discusses how 'Meta-Llama-3-8B-Instruct and Phi-2 have been reported to exhibit data contamination.' The fine-tuning experiments (Section 4.3, Figure 5) directly test memorization effects."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark contamination is the core problem addressed. The paper demonstrates that MBPP/MBPP+ are vulnerable (Figure 1 shows contamination effects) and proposes dynamic generation as a mitigation. The fine-tuning experiment shows DynaCode resists memorization compared to MBPP+."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study using automated code execution."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported despite evaluating 12 LLMs (including commercial APIs like GPT-4o and GPT-3.5-Turbo) across thousands of problems with multiple seeds."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total API spend, or hardware specifications are reported for either the benchmark generation process or the model evaluation."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports mean ± std across 'three times with 5 different random seeds.' Variation is visible (e.g., GPT-4o Unit 2: ±1.4, Unit 4: ±0.3), demonstrating seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly states 'all experiments were conducted three times with 5 different random seeds, and the average results are presented.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "For the main evaluation, temperature=0 is fixed with no search. For the fine-tuning experiments, specific epoch counts and steps are chosen (5 epochs, 1890 steps for GPT-3.5) without reporting what configurations were tried or how these values were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The fine-tuning experiments use specific epoch counts (5 for GPT-3.5, 10 for Llama) and fixed total steps without justifying why these configurations were chosen or whether alternatives were explored."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed in the paper. All comparisons are made by inspecting point estimates and standard deviations. Since no tests are run, multiple comparison correction is structurally inapplicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose DynaCode and evaluate it as both benchmark designers and evaluators. They do not acknowledge this bias or include any independent evaluation of the benchmark's validity."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 7B to 405B parameters (and commercial APIs) are compared without discussing compute differences. The benchmark itself requires generating and executing large numbers of test cases, but compute costs are not reported."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses what DynaCode measures: code generation capability under varying cyclomatic complexity and call-graph structure. Section 4.3 provides error analysis (Table 3) mapping failures to specific capabilities (Problem Understanding, Code Pattern Generation, Context Management). The paper contrasts this with static benchmarks' construct validity problems."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. All models are evaluated via direct single-turn prompting without any agentic framework."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Temporal leakage is the paper's central motivation. They demonstrate that MBPP/MBPP+ (published 2021) are available before model training cutoffs, causing contamination (Figure 1). DynaCode's dynamic generation is designed to prevent temporal leakage. They also add recent LeetCode problems to further mitigate this."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the evaluation setup itself leaks information. For example, the assert statements in prompts (Table 8) provide input-output examples that could serve as hints. Whether the prompt structure provides more information than a real-world coding task is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether base problems from MBPP+ share structural similarities or dependencies. Problems within the same unit share similar complexity, and the same base problems are reused across many nested combinations, but this non-independence is not analyzed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The fine-tuning experiments (Section 4.3, Figure 5) serve as a concrete leakage detection method: by fine-tuning on the benchmark and measuring performance gains, they quantify how much memorization contributes to scores (MBPP+ gains 19pp vs DynaCode gains 3.4pp for GPT-3.5)."
       }
     }
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "claims": [
     {
       "claim": "DynaCode generates approximately 189 million unique nested code generation tasks across 4 units and 16 call-graph structures.",
       "evidence": "Table 1 shows the full breakdown: Unit 1 (164.5M), Unit 2 (3.2M), Unit 3 (2.4M), Unit 4 (19.2M), totaling 189,263,141 problems.",
       "supported": "strong"
     },
     {
       "claim": "LLMs show an average performance drop of 16.8% to 45.7% on DynaCode compared to MBPP+.",
       "evidence": "Table 2 shows drops for all 12 models. GPT-4o drops from 72.2% (MBPP+) to 55.4% (DynaCode, -16.8pp). Meta-Llama-3.1-8B-Instruct drops from 55.6% to 9.9% (-45.7pp).",
       "supported": "strong"
     },
     {
       "claim": "Performance progressively decreases as code complexity (unit) and call-graph complexity (level) increase.",
       "evidence": "Table 2 shows declining scores across units for most models. Figure 3 shows consistent degradation across levels 1-4 for GPT-4o, GPT-3.5-Turbo, WizardLM-2, and Llama-405B.",
       "supported": "strong"
     },
     {
       "claim": "DynaCode's dynamic evaluation strategy effectively mitigates data contamination.",
       "evidence": "Fine-tuning experiments (Figure 5): GPT-3.5 gains 19pp on MBPP+ but only 3.4pp on DynaCode after fine-tuning. Llama-8B gains 42.5pp on MBPP+ but only 13pp on DynaCode. Fine-tuning on unit functions alone drops DynaCode performance.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs perform significantly better on sequential call graphs than complex multi-branch structures.",
       "evidence": "Figure 6 shows 4 models scoring higher on sequential graphs {G1-G4, G8} vs multi-branch graphs {G9-G16}. GPT-4o maintains ~60-80% on sequential but drops to ~30-50% on complex graphs.",
       "supported": "moderate"
     },
     {
       "claim": "Problem Understanding errors increase as code complexity rises, from 64.1% in Unit 1 to 88.8% in Unit 4.",
       "evidence": "Table 3 shows error distribution for GPT-3.5-Turbo: Problem Understanding errors increase monotonically (64.1% → 79.9% → 88.2% → 88.8%) while Code Pattern Generation and Context Management errors decrease.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "DynaCode is a dynamic benchmark generating ~189M unique nested Python code problems by combining base problems from MBPP+ with 16 call-graph structures across 4 complexity levels. Evaluation of 12 LLMs shows 16.8-45.7 percentage point drops compared to MBPP+, with performance degrading consistently as both code and call-graph complexity increase. Fine-tuning experiments demonstrate that DynaCode resists memorization better than static benchmarks, with models gaining much less from training on DynaCode data than on MBPP+. Error analysis reveals that Problem Understanding failures dominate at higher complexity levels, while LLMs perform relatively well on sequential call graphs but struggle with multi-branch dependencies.",
   "red_flags": [
     {
       "flag": "Prompt length confound not addressed",
       "detail": "Higher-complexity call graphs produce longer prompts (more functions, more instructions). The observed performance degradation could be partly attributable to increased prompt length rather than genuine complexity-handling difficulty. This confound is never discussed."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "The paper compares 12 models across 4 units, 4 levels, and 16 call graphs, making dozens of comparative claims (e.g., 'GPT-4o consistently outperforms others') without any statistical significance tests. Given the reported standard deviations, some claimed differences may not be statistically significant."
     },
     {
       "flag": "Self-evaluation bias",
       "detail": "The authors design, implement, and evaluate DynaCode themselves without independent validation. They select which metrics to emphasize, which models to compare, and how to present results. No independent replication or external evaluation is included."
     },
     {
       "flag": "Limited base problem diversity",
       "detail": "The entire benchmark is built from 378 MBPP+ problems plus 40 LeetCode problems. Despite generating 189M combinations, the underlying function implementations are drawn from a narrow set. The paper does not analyze whether this limits the diversity of actual coding challenges tested."
     },
     {
       "flag": "Cyclomatic complexity as sole complexity metric",
       "detail": "Code complexity is classified solely by cyclomatic complexity, which measures control-flow branching. Other complexity dimensions (algorithmic complexity, data structure usage, API usage, string manipulation) are ignored. The paper acknowledges only the node-count limitation, not this metric limitation."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Pass@k metric, foundational to LLM code generation evaluation."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark, the base problem set used by DynaCode."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Introduces EvalPlus and MBPP+, the extended benchmark used as DynaCode's unit function source."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Real-world code generation benchmark focusing on repository-level tasks, contrasted with DynaCode's function-level approach."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["Terry Yue Zhuo", "Minh Chien Vu", "Jenny Chim"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Code generation benchmark with diverse function calls and complex instructions, directly comparable to DynaCode."
     },
     {
       "title": "Top leaderboard ranking = top coding proficiency, always? EvoEval: Evolving coding benchmarks via LLM",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Dynamic benchmark evolution approach using LLMs, addressing similar data contamination concerns as DynaCode."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code benchmark using temporal splits, addressing the same data contamination problem."
     },
     {
       "title": "CRUXEval: A benchmark for code reasoning, understanding and execution",
       "authors": ["Alex Gu", "Baptiste Rozière", "Hugh Leather"],
       "year": 2024,
       "arxiv_id": "2401.03065",
       "relevance": "Code reasoning benchmark evaluating understanding and execution, complementary to code generation evaluation."
     },
     {
       "title": "A careful examination of large language model performance on grade school arithmetic",
       "authors": ["Hugh Zhang", "Jeff Da", "Dean Lee"],
       "year": 2024,
       "arxiv_id": "2405.00332",
       "relevance": "Documents data contamination in LLMs including Meta-Llama-3-8B-Instruct, directly cited as evidence for contamination."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["Xinyi Hou", "Yanjie Zhao", "Yue Liu"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLMs for software engineering, providing broader context for code generation evaluation."
     },
     {
       "title": "DyVal: Graph-informed dynamic evaluation of large language models",
       "authors": ["Kaijie Zhu", "Jiaao Chen", "Jindong Wang"],
       "year": 2023,
       "arxiv_id": "2309.17167",
       "relevance": "Graph-based dynamic evaluation approach that inspired DynaCode's methodology for generating dynamic benchmarks."
     },
     {
       "title": "EvoCodeBench: An evolving code generation benchmark aligned with real-world code repositories",
       "authors": ["Jia Li", "Ge Li", "Xuanming Zhang"],
       "year": 2024,
       "arxiv_id": "2404.00599",
       "relevance": "Evolving code benchmark focused on real-world repository alignment, addressing benchmark staleness."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Released benchmark tool that practitioners can use to evaluate LLMs on code generation, though limited to Python function composition."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms the expected finding that LLMs struggle with increased complexity; the magnitude of performance drops (up to 45.7pp) is noteworthy but not fundamentally surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; purely a benchmark evaluation paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Shows popular benchmarks like MBPP are contaminated and unreliable, but data contamination in LLM benchmarks is already well-established."
     },
     "demo_ability": {
       "score": 2,
       "justification": "GitHub repository released with benchmark generation and evaluation code; users could run it on their own models."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates well-known models (GPT-4o, DeepSeek-V3, Llama-3) but paper is from UESTC/Drexel, not a major AI lab."
     }
   }
 }