ai-research-survey

scan.json (26444B)

---

 {
   "paper": {
     "title": "AnCoder: Anchored Code Generation via Discrete Diffusion Models",
     "authors": ["Anton Xue", "Litu Rout", "Constantine Caramanis", "Sanjay Shakkottai"],
     "year": 2026,
     "venue": "Preprint",
     "arxiv_id": "2602.17688"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided anywhere in the paper, footnotes, or appendix. No mention of a GitHub link or Zenodo archive."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: HumanEval (MIT License) and MBPP (Apache 2.0), and training data from OpenCoder (opc-fineweb-code, opc-annealing-corpus). Appendix A.2 explicitly states 'All datasets used in this work are publicly available and licensed for research use.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is provided. The paper mentions using AdamW and cosine schedules but does not specify software versions, library dependencies, or hardware-software stack details beyond mentioning the Vista GPU Cluster."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided. While training hyperparameters are described in Section 5 and Appendix A, there are no concrete instructions a researcher could follow to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1, 2, 3, and 5 are reported as point estimates (e.g., '5.45%' Pass@1) with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'AnCoder consistently outperforms the MDLM and anchoring baselines' (Section 5.1) based on comparing raw numbers without any statistical significance tests (no p-values, no bootstrap tests, no paired tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 reports improvements with baseline context: 'Pass@1 from 3.29% to 5.45% (+2.16 absolute, +65.7% relative)' and 'syntactic validity from 65.58% to 74.72% (+9.14 absolute, +13.9% relative).' These provide both absolute and relative effect sizes with clear baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of samples generated per problem (20 for HumanEval, 10 for MBPP) or for the benchmark sizes. No power analysis or discussion of whether these sample sizes are sufficient to detect meaningful differences."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. It is not stated whether results are from single runs or averaged over multiple seeds. All tables show single-point estimates only."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares AnCoder against MDLM (494M and 809M), the AR baseline Qwen2.5-Coder-0.5B, and ablated anchoring variants (Null, Keyword, Identifier). This provides both prior-work and ablation baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares against MDLM [38] (2024) and Qwen2.5-Coder-0.5B, but omits comparison with other recent diffusion-based code generation models mentioned in the related work: DiffuCoder [16], Dream-Coder [48], CoDA [9], Mercury [22], and Gemini Diffusion [13]. These are all from 2025 and represent the state of the art for DLM code generation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 and Figure 6/Table 5 present a systematic ablation of the anchor supervision weight γ across three anchoring strategies (AnchorTree, Keyword, Identifier). Additionally, Table 1 effectively ablates anchoring strategy choice (Null vs. Keyword vs. Identifier vs. AnchorTree)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports two metrics: syntactic validity (Syntax %) and functional correctness (Pass@1 %) across all experiments (Tables 1, 2, 3, 5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code quality is included. Evaluation is entirely automated via syntactic parsing and unit test execution. Given claims about code quality and structural correctness, human evaluation of readability and maintainability would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "HumanEval and MBPP are standard held-out benchmarks with predefined test splits. The paper also separates training/validation sets for its own training stages (e.g., 99%-1% split for mid-training, 95%-5% for SFT)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregate numbers (overall Syntax% and Pass@1%) are reported for HumanEval and MBPP. No per-problem, per-difficulty, or per-category breakdowns are provided. Qualitative examples in Appendix A.4 show individual cases but do not constitute systematic per-category analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.4 provides detailed qualitative analysis of failure cases, showing how MDLM and weaker anchoring strategies produce syntactically valid but semantically incorrect code, with specific examples from HumanEval and MBPP."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper honestly reports that 'all diffusion models fall short of the AR baseline in terms of syntactic validity and pass@1 on both benchmarks' (Section 5.1). The ablation also shows that performance can degrade with certain γ values (Table 5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that AnCoder is 'a family of models showing that structurally anchored diffusion offers a parameter-efficient path to high-quality code generation.' The results in Table 1 support this: AnCoder (809M) outperforms MDLM baselines. The 'parameter-efficient' claim is supported by the 809M model outperforming a comparable 809M MDLM. The abstract does not overclaim relative to AR models."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about AnchorTree improving code generation quality. These are supported by controlled ablation: all 809M models use the same architecture, training recipe, and data; the only variable is the anchoring strategy. Section 4.2 also provides a controlled unmasking experiment (Figure 5) demonstrating the causal effect of AST ancestry on prediction quality."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only on Python code (HumanEval and MBPP are Python-only benchmarks) and a single model scale (809M parameters), but the title and conclusion use broad language like 'Anchored Code Generation' and 'high-quality code generation' without qualifying these claims to Python at 809M scale. Section A.5 acknowledges 'AnchorTree is explicitly focused on code hierarchies extracted from the abstract syntax tree' but does not bound to Python specifically."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements. For example, the remask rate (0.1 for AnCoder vs. 0.0 for MDLM) is a confound that is not adequately controlled. Section 5.1 notes this difference but does not disentangle the contribution of remasking from anchoring. No threats-to-validity or confound analysis is provided."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies Qwen2.5-Coder-0.5B [21] as the initialization model (Section 5, 'Architecture'). Since this is a model they train from scratch atop this initialization, the base model is adequately specified with its full name and citation."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use LLM prompting in the traditional sense. AnCoder is a diffusion model that takes function signatures/docstrings as conditioning inputs, which are from the standard benchmarks (HumanEval, MBPP). Prompting in the API/chat sense is not applicable."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 reports: learning rate 10^-4, batch size 512, cosine LR scheduler, warmup ratio 0.01, cosine noise schedule, sampling temperature 0.8, remask rate 0.1, context length L=1024, denoising steps T in {256, 512, 1024, 2048}. Anchor weight γ values are specified per strategy. Training steps per stage are also listed."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. AnCoder is a diffusion language model evaluated on benchmark code generation tasks without any agent scaffold, tool use, or iterative prompting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 documents the three-stage training pipeline with specific datasets, sample counts, and token counts for each stage (Table 4). The SFT stage filtering to Python-only is described. The AST-based anchor labeling preprocessing is described in Sections 4.1 and 4.3. Context length truncation/padding to L=1024 is stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.5 is titled 'Limitations' and provides substantive discussion of limitations. Section 6 (Discussion) also discusses design space limitations and alternative approaches."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations in Appendix A.5 are relatively generic: 'the development of anchored diffusion models is still nascent, and there exists a large search space of architecture design' and 'AnchorTree is explicitly focused on code hierarchies extracted from the abstract syntax tree, which may limit its ability to reason about dynamic execution behavior.' These do not address specific threats to the validity of the reported results (e.g., the remask rate confound, single-run results, Python-only evaluation, HumanEval contamination)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its claims to Python, to 809M scale, or to the specific benchmarks used. The limitations section discusses future directions but not specific scope exclusions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (generated code samples, per-problem pass/fail results, training logs) is made available. Only aggregate metrics in tables are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection/generation procedure is well-described: benchmarks are HumanEval (164 problems) and MBPP (974 problems), with 20 and 10 samples per problem respectively. Training datasets are documented in Appendix A.1 with sources, sizes, and domains."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data sources are standard public benchmarks and public code corpora."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full training pipeline is documented across three stages (pretraining, mid-training, SFT) with dataset sizes, training steps, and splits at each stage (Appendix A.1). The AST-based anchor labeling pipeline is described in Sections 4.1 and 4.3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This research has been supported by NSF Grants 2019844 and 2112471, the UT Austin Machine Learning Lab, and computing support on the Vista GPU Cluster through the Center for Generative AI (CGAI) and the Texas Advanced Computing Center (TACC) at UT Austin.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with The University of Texas at Austin. The paper does not evaluate any commercial product from UT Austin, so there is no obvious undisclosed conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from NSF and UT Austin computing resources. Neither has a financial stake in whether AnchorTree outperforms baselines."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The model is initialized from Qwen2.5-Coder-0.5B, but no training data cutoff date for Qwen is stated. The paper's own training data sources are described but without temporal cutoff information relative to the benchmark creation dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between training data and HumanEval/MBPP test problems. The training data comes from OpenCoder's web-scale code corpus which could plausibly contain HumanEval/MBPP solutions."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021 and MBPP in 2021. Qwen2.5-Coder was released in 2024 and the OpenCoder data is web-scale. Both benchmarks' solutions are widely available online. The paper does not discuss contamination risk at all, despite this being a well-known issue for these benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or wall-clock time is reported. The paper studies scaling denoising steps T from 256 to 2048 but does not report the time or compute cost associated with each setting, which is critical for practical deployment of diffusion models for code generation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total GPU hours, training time, or compute budget is stated. The paper mentions using the Vista GPU Cluster and TACC but does not quantify the computational resources consumed across three training stages totaling ~418K optimization steps."
       }
     }
   },
   "claims": [
     {
       "claim": "AnCoder with AnchorTree improves Pass@1 on HumanEval from 3.29% to 5.45% (+65.7% relative) over 809M MDLM baseline.",
       "evidence": "Table 1, Section 5.1. Direct comparison of 809M models with identical architecture, training recipe, and data.",
       "supported": "moderate"
     },
     {
       "claim": "AnCoder with AnchorTree improves Pass@1 on MBPP from 6.34% to 9.10% (+43.5% relative) over 809M MDLM baseline.",
       "evidence": "Table 1, Section 5.1. Same controlled comparison setup.",
       "supported": "moderate"
     },
     {
       "claim": "Soft hierarchical anchoring (AnchorTree) outperforms hard anchoring (Keyword-only, Identifier-only) strategies.",
       "evidence": "Table 1 shows AnchorTree > Keyword > Identifier on Pass@1 for both benchmarks. Ablation in Table 5/Figure 6 confirms the pattern.",
       "supported": "moderate"
     },
     {
       "claim": "Unmasking AST ancestors of a target position improves prediction quality more than unmasking random positions.",
       "evidence": "Figure 5, Section 4.2. Controlled experiment on pretrained DLM with 2000 test samples showing in-out AST unmasking outperforms random and out-in orderings.",
       "supported": "strong"
     },
     {
       "claim": "Anchored models benefit more from increased denoising steps (test-time compute scaling) than non-anchored models.",
       "evidence": "Tables 2 and 3 show steeper performance improvement curves for anchored models as T increases from 256 to 2048.",
       "supported": "moderate"
     },
     {
       "claim": "All diffusion models fall short of the autoregressive baseline on both benchmarks.",
       "evidence": "Table 1 shows AR (494M) achieves 26.83/36.55 Pass@1 on HumanEval/MBPP vs. AnCoder AnchorTree (809M) at 5.45/9.10.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "AnCoder introduces AnchorTree, a soft anchoring framework that uses abstract syntax tree hierarchy to guide discrete diffusion language models for code generation. On HumanEval and MBPP, AnCoder with AnchorTree improves Pass@1 by 65.7% and 43.5% relative over MDLM baselines at 809M parameters, while also improving syntactic validity. However, all diffusion models still substantially underperform a 494M autoregressive baseline (Qwen2.5-Coder-0.5B), with AnCoder achieving 5.45% vs. 26.83% Pass@1 on HumanEval. The paper demonstrates that AST-based hierarchical information is useful for diffusion denoising through a controlled unmasking experiment.",
   "red_flags": [
     {
       "flag": "Remask rate confound",
       "detail": "AnCoder uses a remask rate of 0.1 while the MDLM baseline uses 0.0. Section 5 states this explicitly but does not adequately disentangle the contribution of remasking from anchoring. The 'Null' anchoring variant (no anchoring, but with remasking) already shows substantial improvement over MDLM, suggesting remasking may account for much of the gain."
     },
     {
       "flag": "No error bars or variance reporting",
       "detail": "All results are reported as single point estimates. For Pass@1 values in the 3-9% range, the differences between methods (often 1-3 percentage points) could plausibly be within random variation, especially without knowing whether results are from single or multiple runs."
     },
     {
       "flag": "Missing contemporary DLM baselines",
       "detail": "The paper discusses DiffuCoder, Dream-Coder, CoDA, Mercury, and Gemini Diffusion in the related work but only compares against MDLM. These contemporary methods are directly relevant competitors that are omitted from the empirical evaluation."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "HumanEval and MBPP solutions are widely available online. The model is initialized from Qwen2.5-Coder (trained on web-scale data) and further trained on OpenCoder's web-scale corpus. No contamination analysis is performed."
     },
     {
       "flag": "Absolute performance levels are very low",
       "detail": "The best diffusion model achieves 5.45% Pass@1 on HumanEval vs. 26.83% for the AR baseline. While the relative improvement from anchoring is notable, the absolute performance level raises questions about practical relevance. The paper's framing emphasizes relative gains while the AR gap is enormous."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark, a foundational evaluation tool for LLM code generation widely used across the field."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark for evaluating code generation capabilities of language models."
     },
     {
       "title": "DiffuCoder: Understanding and improving masked diffusion models for code generation",
       "authors": ["Shansan Gong", "Ruixiang Zhang", "Huangjie Zheng"],
       "year": 2025,
       "arxiv_id": "2506.20639",
       "relevance": "Contemporary diffusion-based code generation model that improves masked diffusion for code, directly relevant as an alternative approach."
     },
     {
       "title": "Beyond autoregression: An empirical study of diffusion large language models for code generation",
       "authors": ["Chengze Li", "Yitong Zhang", "Jia Li"],
       "year": 2025,
       "arxiv_id": "2509.11252",
       "relevance": "Empirical comparison of diffusion vs. autoregressive LLMs for code generation, establishing DLM capabilities and limitations."
     },
     {
       "title": "Mercury: Ultra-fast language models based on diffusion",
       "authors": ["Samar Khanna", "Siddhant Kharbanda", "Shufan Li"],
       "year": 2025,
       "arxiv_id": "2506.17298",
       "relevance": "Diffusion-based language model focusing on speed, relevant to understanding the DLM-for-code landscape."
     },
     {
       "title": "Qwen2.5-coder technical report",
       "authors": ["Binyuan Hui", "Jian Yang", "Zeyu Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Technical report for the autoregressive model used as AnCoder's initialization backbone and primary AR baseline."
     },
     {
       "title": "Simple and effective masked diffusion language models",
       "authors": ["Subham Sahoo", "Marianne Arriola", "Yair Schiff"],
       "year": 2024,
       "relevance": "MDLM is the primary baseline method in this paper; foundational work on masked discrete diffusion for language."
     },
     {
       "title": "Large language diffusion models",
       "authors": ["Shen Nie", "Fengqi Zhu", "Zebin You"],
       "year": 2025,
       "arxiv_id": "2502.09992",
       "relevance": "Major work on scaling diffusion language models, relevant to understanding DLM capabilities and the AR-DLM performance gap."
     },
     {
       "title": "Constrained decoding of diffusion LLMs with context-free grammars",
       "authors": ["Niels Mündler", "Jasper Dekoninck", "Martin Vechev"],
       "year": 2025,
       "arxiv_id": "2508.10111",
       "relevance": "Complementary approach to ensuring syntactically valid code from diffusion models via constrained decoding."
     },
     {
       "title": "Anchored diffusion language model",
       "authors": ["Litu Rout", "Constantine Caramanis", "Sanjay Shakkottai"],
       "year": 2025,
       "relevance": "Foundational work introducing anchored diffusion for language models, directly extended by this paper to code generation."
     },
     {
       "title": "TreeDiff: AST-guided code generation with diffusion LLMs",
       "authors": ["Yiming Zeng", "Jinghan Cao", "Zexin Li"],
       "year": 2025,
       "arxiv_id": "2508.01473",
       "relevance": "Most closely related work: also uses AST structure for diffusion code generation but modifies the forward noising schedule rather than denoising."
     },
     {
       "title": "Dream-coder 7b: An open diffusion language model for code",
       "authors": ["Zhihui Xie", "Jiacheng Ye", "Lin Zheng"],
       "year": 2025,
       "arxiv_id": "2509.01142",
       "relevance": "Open diffusion language model for code at larger scale (7B), relevant as a contemporary DLM code generation baseline."
     }
   ]
 }