ai-research-survey

scan.json (20741B)

---

 {
   "paper": {
     "title": "Constrained Decoding of Diffusion LLMs with Context-Free Grammars",
     "authors": ["Niels Mündler", "Jasper Dekoninck", "Martin Vechev"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2508.10111"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository linked: https://github.com/eth-sri/constrained-diffusion and project page https://constrained-diffusion.ai provided in the paper header."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available HumanEval-X dataset and describes synthetic benchmark generation procedures for JSON and SMILES. The code repository would contain these benchmarks. The base datasets (HumanEval-X, NousResearch json-mode-eval) are public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions the implementation is ~7000 lines Python and ~5500 lines Rust but does not provide requirements.txt, Dockerfile, or detailed dependency/version specifications beyond mentioning Rdkit."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The paper describes the method and experiments but does not include a 'Reproducing Results' section or specific commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: 'We compute confidence intervals at 95%, boldface the best method, and underline all methods over which the increase is not significant. The usual size of the confidence interval is 1% to 2%.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses 95% confidence intervals to determine significance, underlining results where the improvement is not statistically significant. This serves as a significance testing mechanism."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute improvements are reported with baseline context throughout, e.g., 'an absolute increase of 5.2%, 22.5%, and 31.5% for 1-MRI, 2-MRI, and 3-MRI' and 'up to 7%' functional correctness improvement. Full baseline and method numbers in tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for using 164 HumanEval tasks, 272 JSON instances, or 167 SMILES instances. No power analysis discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and Appendix D.2 state all methods were run 4 times with different seeds (0-4) and 95% confidence intervals are reported, indicating variance across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Vanilla (unconstrained) decoding is the primary baseline, and Con.- (constrained without completion sampling) serves as an ablation baseline. Results compared across 5 infilling models and 4 diffusion models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes no prior work supports CFG constraining for DLMs or MRI. The comparison is against unconstrained decoding on state-of-the-art models (DREAM 7B, DiffuCoder 7B, LLaDA 8B, etc., all 2024-2025). Suresh et al. [47] (Dingo) is discussed but only handles regular languages, not CFGs."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates: Con.- vs Con. (with/without completion sampling), varying number of infilling regions (1-MRI, 2-MRI, 3-MRI), varying model sizes (1.3B to 33B), and Appendix D mentions ablation on diffusion steps."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two main metrics: syntactic correctness (Syntax) and functional correctness (Functional/pass@1). Runtime overhead is also measured as a third metric."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant here — syntactic correctness is deterministic (grammar adherence) and functional correctness is measured via test cases."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a learning/training paper. The method is an inference-time algorithm applied to existing benchmarks; there is no train/dev/test split concern."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by model (5 infilling + 4 diffusion), by task (C++, JSON, SMILES), by number of infilling regions (1-MRI, 2-MRI, 3-MRI), and runtime per model/task."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses timeout failures, notes that 'only 0.7% of valid completions do not appear in the first 50 selected LLM proposals,' discusses SMILES functional correctness being very low (1.5% average), and mentions DreamCoder achieving only 19% syntactic correctness on C++ even with constraints."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "SMILES functional correctness shows negligible improvement (0.2% average increase). The paper reports cases where constrained decoding cannot help (model timeouts, poor functional quality despite syntactic validity)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 'near-perfect syntactic correctness' supported by Tables 1-2 (96-100% with completion), 'improving functional correctness' supported (up to 7%), and 'computational overhead remains practical' supported by runtime tables showing median 30-125% overhead."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that constrained decoding improves correctness is justified by controlled single-variable manipulation: same models, same seeds, same benchmarks, with/without the constraining algorithm. The ablation Con.- vs Con. isolates the completion sampling component."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to tested settings: specific models (named with sizes), specific tasks (C++, JSON, SMILES), specific benchmarks. The title specifies 'Context-Free Grammars' and the paper does not claim applicability beyond CFGs."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for observed results. For example, no discussion of whether functional correctness improvements could be due to other factors, or whether the grammar overapproximation affects results systematically."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are named with parameter counts (e.g., 'STARCODER2 7B', 'DREAM 7B', 'LLADA 8B') but no specific version snapshots, checkpoint dates, or HuggingFace model IDs are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Example prompts for all three tasks (C++, JSON, SMILES) are shown in Figures 7-10 in the appendix, and generation/verification prompts for synthetic data creation in Figures 11-14."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.2 reports: temperature (1 for MRI, 0.2 for DLM), sampling algorithms (greedy for MRI, entropy/low-confidence for DLM), diffusion steps (32), max tokens (256), timeout (300s), rejection limit (100), seeds (0-4)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a constrained decoding algorithm applied at inference time."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.3 documents dataset construction in detail: MRI span removal procedure (5-100 chars, resampling), JSON schema cleaning/filtering, SMILES generation with three filtering steps (validity, solvability, deduplication), with resulting instance counts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section. Some limitations are mentioned in passing (grammar overapproximation in Appendix E, C++ grammar not covering all features) but no consolidated discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity discussed. The grammar coverage limitation and overapproximation are mentioned but not framed as threats to validity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do NOT show. The paper does not discuss limitations of the benchmarks used or bound claims to specific settings explicitly."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental outputs (model generations, per-instance results) are not provided. Only aggregate statistics in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.3 describes dataset construction procedures for all three tasks in detail, including filtering criteria and resulting counts."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks and synthetic generation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from HumanEval-X to MRI tasks (span removal, filtering invalid solutions) and from Gemini generation to JSON/SMILES benchmarks (three filtering steps with counts: 272 JSON, 167 SMILES) is documented in Appendix D.3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors listed as ETH Zurich, Department of Computer Science. No commercial product is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any of the 9 models evaluated. The paper uses HumanEval (published 2021) with models trained after 2021."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval examples appeared in training data of the evaluated models, despite HumanEval being a widely-known public benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021; all evaluated models were trained after this date. No discussion of contamination risk. The synthetic JSON and SMILES benchmarks would not have this issue, but C++ HumanEval does."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Runtime overhead reported in detail: Tables 3-4 show per-token and per-completion overhead in ms and percentages for all models and settings. Median overhead ranges from 30% (DLM) to 125% (MRI)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of total GPU hours, hardware used for experiments, or total compute budget. Only relative overhead percentages are reported."
       }
     }
   },
   "claims": [
     {
       "claim": "Constrained decoding with completion sampling achieves near-perfect syntactic correctness (95.8-100%) across all settings.",
       "evidence": "Tables 1 and 2 show Con. achieving 83-100% syntax in MRI and 99.2-100% in DLM settings (Section 4.2-4.3).",
       "supported": "strong"
     },
     {
       "claim": "Constrained decoding improves functional correctness by up to 7% over unconstrained generation.",
       "evidence": "Table 1 shows functional correctness improvements averaging 2.8% for MRI; Table 2 shows DREAM 7B JSON improving by 6.9% (Section 4.2-4.3).",
       "supported": "strong"
     },
     {
       "claim": "The computational overhead remains practical, with median inference time less than doubling on average.",
       "evidence": "Tables 3-4: MRI median overhead 125%, DLM median overhead 30%. For 7B models, overhead is ~100% (Section 4.2-4.3, Appendix D.4).",
       "supported": "strong"
     },
     {
       "claim": "Only 0.7% of valid completions do not appear in the first 50 selected LLM proposals.",
       "evidence": "Stated in Section 3.2 without further detail on how this was measured.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper presents the first constrained decoding method for diffusion language models that handles context-free grammars, enabling guaranteed syntactic correctness for C++, JSON, and SMILES generation. Across 9 models and 3 tasks, constrained decoding with completion sampling achieves 95.8-100% syntactic correctness (vs. 11-94% unconstrained) with modest runtime overhead (30-125% median). Functional correctness also improves by 2-7% since syntactically invalid outputs are prevented. The method generalizes across prefix, fill-in-the-middle, multi-region infilling, and diffusion LLM generation paradigms.",
   "red_flags": [
     {
       "flag": "No contamination analysis for HumanEval",
       "detail": "HumanEval was published in 2021 and is widely available online. All evaluated models were trained after 2021 and likely saw these examples during training. The paper does not discuss this contamination risk. While this affects functional correctness comparisons more than syntactic correctness (the main contribution), it undermines the validity of pass@1 results."
     },
     {
       "flag": "Synthetic benchmarks generated by Gemini",
       "detail": "The JSON and SMILES benchmarks were generated by Gemini-2.5-Pro. While filtering steps are applied, the quality and diversity of these synthetic benchmarks is not independently validated. No analysis of benchmark difficulty distribution or comparison to human-created benchmarks."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section. Grammar coverage limitations (e.g., no C++ templates) and the overapproximation of infilling region sizes are mentioned only in passing."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark paper, foundational code generation evaluation used in this work."
     },
     {
       "title": "Guiding LLMs The Right Way: Fast, Non-invasive Constrained Generation",
       "authors": ["Luca Beurer-Kellner", "Marc Fischer", "Martin Vechev"],
       "year": 2024,
       "relevance": "Prior constrained decoding method for autoregressive LLMs that this work generalizes to diffusion models."
     },
     {
       "title": "SynCode: LLM Generation with Grammar Augmentation",
       "authors": ["Shubham Ugare", "Tarun Suresh"],
       "year": 2024,
       "arxiv_id": "2403.01632",
       "relevance": "Grammar-augmented constrained decoding for autoregressive LLMs, a key prior approach."
     },
     {
       "title": "Dingo: Constrained inference for diffusion llms",
       "authors": ["Tarun Suresh", "Debangshu Banerjee"],
       "year": 2025,
       "arxiv_id": "2505.23061",
       "relevance": "Most closely related work: constrains diffusion LLMs but only to regular languages, not CFGs."
     },
     {
       "title": "Large Language Diffusion Models",
       "authors": ["Shen Nie", "Fengqi Zhu"],
       "year": 2025,
       "arxiv_id": "2502.09992",
       "relevance": "LLaDA model paper, one of the diffusion LLMs evaluated in this work."
     },
     {
       "title": "DiffuCoder: Understanding and Improving Masked Diffusion Models for Code Generation",
       "authors": ["Shansan Gong", "Ruixiang Zhang"],
       "year": 2025,
       "arxiv_id": "2506.20639",
       "relevance": "DiffuCoder model paper, one of the diffusion LLMs evaluated; focuses on code generation with diffusion."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han"],
       "year": 2025,
       "relevance": "Contamination-aware code benchmark, relevant to evaluation methodology for LLM code generation."
     },
     {
       "title": "BaxBench: Can LLMs generate correct and secure backends?",
       "authors": ["Mark Vero", "Niels Mündler"],
       "year": 2025,
       "relevance": "LLM code generation benchmark for backend systems, from same research group."
     },
     {
       "title": "Constrained Decoding for Fill-in-the-middle Code Language Models via Efficient Left and Right Quotienting of Context-sensitive Grammars",
       "authors": ["Daniel Melcer", "Nathan Fulton"],
       "year": 2024,
       "arxiv_id": "2402.17988",
       "relevance": "Extends constrained decoding to FIM with context-sensitive features; directly extended by this work."
     },
     {
       "title": "A Survey on Large Language Models for Code Generation",
       "authors": ["Juyong Jiang", "Fan Wang"],
       "year": 2024,
       "relevance": "Survey of LLM code generation approaches, relevant to the broader survey scope."
     }
   ]
 }