ai-research-survey

scan.json (23564B)

---

 {
   "paper": {
     "title": "Guiding LLMs The Right Way: Fast, Non-Invasive Constrained Generation",
     "authors": ["Luca Beurer-Kellner", "Marc Fischer", "Martin Vechev"],
     "year": 2024,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2403.06988",
     "doi": "10.48550/arXiv.2403.06988"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DOMINO, a minimally-invasive constrained decoding algorithm, enforces context-free grammar constraints during LLM generation without accuracy loss and often with speedup (up to 2.71x) over unconstrained decoding. Naive constrained decoding methods cause token misalignment that can reduce task accuracy by up to 11 percentage points (GSM8K with GUIDANCE). The paper introduces speculative decoding for constrained generation and pre-computed subterminal trees for efficient vocabulary-grammar alignment.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. No mention of code release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: GSM8K and CoNLL-2003, which are standard public datasets. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions NVIDIA A100 40GB and H100 80GB GPUs and the transformers and llama.cpp backends, but does not provide a requirements.txt, Dockerfile, or detailed library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 2-4 report only point estimates (e.g., accuracy 0.415, throughput 1.77x) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims DOMINO outperforms baselines but provides no statistical significance tests — comparisons are made by directly comparing numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative performance (e.g., '1.77× throughput', accuracy changes from 0.415 to 0.308) with baseline context, allowing the reader to gauge effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "400 test samples are used from each dataset and 100 repetitions for throughput experiments, but no justification for these choices is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations or variance across runs are reported. Throughput experiments use 100 repetitions but only report aggregate numbers without spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against unconstrained generation, GUIDANCE (template and CFG variants), and llama.cpp grammars across all experiments (Tables 2, 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GUIDANCE, llama.cpp, and SYNCHROMESH are contemporary constrained decoding methods. Also references GCD (2023) and PICARD (2021). Table 1 provides a comprehensive comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 ablates the lookahead parameter k. Figure 5 studies the number of speculative tokens s. The paper also compares opportunistic masking vs speculative decoding modes."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports task accuracy, well-formedness, perplexity, and throughput (performance impact) in Table 2. Additional throughput comparisons in Table 3."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant here — the claims are about format validity, task accuracy on benchmarks, and throughput, all of which are objectively measurable."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses 400 test samples from the test splits of GSM8K and CoNLL-2003, with few-shot demonstrations from the training split (§4.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (GSM8K, CoNLL-2003), per model (Mistral 7B, Llama-2 13B), and per grammar type (JSON, JSON with schema, C, XML, Fixed Template) in Tables 2-3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure cases: Figure 1 shows how naive constraining distorts tokenization, Figure 2 shows template-induced perplexity explosion, Table 4 shows k=0 and k=1 cause accuracy drops, and the paper notes C grammar is hardest for speculative decoding."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that C code generation incurs overhead (0.78x) and speculative decoding is not effective for free-form JSON (§4.3). Lower k values cause severe accuracy degradation (Table 4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (1) existing methods impair accuracy, (2) DOMINO enforces constraints with subword alignment, (3) virtually no overhead and up to 2x speedup. All are supported by Tables 2-4 and Figure 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about token misalignment causing accuracy drops, supported by controlled ablations: same prompts, same models, same datasets, varying only the constraining method. The lookahead ablation (Table 4) provides additional controlled evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests on only two LLMs (Mistral 7B, Llama-2 13B) and two task-accuracy datasets (GSM8K, CoNLL-2003), but the title and abstract make general claims about constrained generation for LLMs without bounding to these specific models or tasks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for accuracy differences (e.g., could the accuracy changes be due to prompt sensitivity rather than tokenization misalignment specifically?). No threats-to-validity or robustness checks."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures what it claims: task accuracy on GSM8K/CoNLL-2003, throughput in tokens/second, and format validity. No proxy gap exists — measurements match claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies Mistral 7B (citing Jiang et al., 2023) and Llama-2 13B (citing Touvron et al., 2023c). These are specific model versions with known architectures and weights."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C and D provide the actual grammars and prompts used for all experiments. Full prompt text and JSON format examples are included."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature 1.0 is stated for throughput experiments (§4.3), max tokens 128, 5-shot prompting for accuracy experiments. The speculative decoding parameter s is studied in Figure 5."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. DOMINO is a decoding-time algorithm, not an agent."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how few-shot demonstrations were constructed (manually creating JSON responses from training split examples, §4.1), the 400 test sample subset, and the prompt format (Appendix D)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper has only a brief Impact Statement that does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what settings or models are NOT covered. It evaluates on only 2 models and 2 accuracy benchmarks but does not acknowledge these scope limitations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (generated outputs, token-level logs, per-sample results) are provided. Only aggregate tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the data sources (GSM8K test split, CoNLL-2003 test split), the sampling procedure (400 test samples, 5 prompts per workload), and the experimental protocol (100 repetitions, 10 warmup)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: prompts are constructed with 5-shot demonstrations from training split, JSON format responses are manually created, models generate output under various constraints, outputs are evaluated for accuracy, validity, and perplexity."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section states: 'This work has received funding from the Swiss State Secretariat for Education, Research and Innovation (SERI)' under the SAFEAI grant."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with ETH Zurich, Department of Computer Science. No commercial product is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is SERI (Swiss government research agency) via an ERC-style grant for AI safety research. The funder has no financial stake in DOMINO's performance."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. The first author is also an author of LMQL, one of the tools compared against, which is not disclosed as a potential conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for Mistral 7B or Llama-2 13B."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GSM8K or CoNLL-2003 examples appeared in the models' training data. Both benchmarks predate these models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021) and CoNLL-2003 (2002) were both available online before Mistral and Llama-2 were trained, creating contamination risk. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively reports throughput (tokens/second) and relative performance overhead for all methods. Figure 5 shows tokens/second for different configurations. Table 3 reports relative throughput vs unconstrained baseline."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions GPU types (A100, H100) but does not state total GPU hours, wall-clock time for experiments, or total computational budget. Precomputation time (1-5s, C at ~20s) is noted but total experiment budget is not."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No reporting of results across multiple random seeds. The paper uses temperature 1.0 for throughput but does not report seed sensitivity for accuracy experiments."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 100 repetitions per configuration for throughput experiments (§4.3) and 10 warmup repetitions. For accuracy, 400 test samples are evaluated but the number of runs is not explicitly stated (appears to be single-run with greedy/sampled decoding)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The lookahead k and speculative tokens s are studied but the selection process for other settings is not documented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically studies the key parameters (k in Table 4, s in Figure 5) rather than just reporting the best configuration. The choice of k=∞ is justified by the ablation showing lower values degrade accuracy."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created DOMINO and are also authors of LMQL (one comparison point). They implement GUIDANCE programs and llama.cpp baselines themselves. No acknowledgment of self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core contribution is precisely about the compute-performance tradeoff. Table 3 compares throughput across methods. Figure 5 shows performance as a function of speculative tokens. The precomputation cost is noted (1-5s, C at 20s)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GSM8K math reasoning and CoNLL-2003 NER are appropriate benchmarks for measuring the impact of constrained decoding on general task accuracy. These are relatively narrow task types."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in this work. DOMINO is a decoding-time algorithm applied directly to LLM inference."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021) and CoNLL-2003 (2002) predate the models used. No discussion of whether models may have seen these benchmarks during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 5-shot demonstrations or prompting format leaks information about expected answers."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training data and test examples for the models used."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used or discussed."
       }
     }
   },
   "claims": [
     {
       "claim": "Naive constrained decoding methods significantly impair task accuracy due to token misalignment, reducing GSM8K accuracy from 41.5% to 30.8% (Mistral 7B with GUIDANCE).",
       "evidence": "Table 2 shows accuracy drops for GUIDANCE across all model/dataset combinations. The largest drop is 11 percentage points on GSM8K with GUIDANCE on Llama-2 13B (0.262 → 0.152).",
       "supported": "strong"
     },
     {
       "claim": "DOMINO achieves minimally-invasive constraining with no accuracy loss compared to unconstrained generation.",
       "evidence": "Table 2 shows DOMINO matching or slightly exceeding unconstrained accuracy on all 4 model/dataset combinations (e.g., 0.418 vs 0.415 on GSM8K Mistral 7B).",
       "supported": "moderate"
     },
     {
       "claim": "DOMINO achieves up to 2.71x throughput over unconstrained generation via speculative decoding.",
       "evidence": "Table 2 reports 2.71x for CoNLL-2003 with Llama-2 13B. Table 3 shows up to 1.91x for fixed templates with Llama-2 13B.",
       "supported": "moderate"
     },
     {
       "claim": "Template-based constrained generation causes perplexity explosion and different output compared to unconstrained generation.",
       "evidence": "Figure 2 shows template-based output has perplexity 24.50-26.75 vs 4.17 for unconstrained, and naturalized template output reaches 49.39 perplexity.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance",
       "detail": "All accuracy and throughput numbers are point estimates with no variance, confidence intervals, or statistical tests despite claiming superiority over baselines."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats to validity, or scope boundaries section despite making broad claims about constrained decoding for LLMs based on only 2 models."
     },
     {
       "flag": "Self-comparison bias",
       "detail": "The first author is also an author of LMQL. The authors implement all baselines (GUIDANCE programs, llama.cpp grammars) themselves without acknowledging potential bias in baseline implementation quality."
     },
     {
       "flag": "Very narrow model evaluation",
       "detail": "Only two relatively small models (7B and 13B) are tested for accuracy. No evaluation on larger models or instruction-tuned variants where constrained decoding behavior may differ significantly."
     },
     {
       "flag": "Contamination not addressed",
       "detail": "GSM8K and CoNLL-2003 predate both models, creating contamination risk that could affect absolute accuracy numbers (though relative comparisons between decoding methods are less affected)."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "relevance": "Codex/HumanEval benchmark foundational to LLM code generation evaluation."
     },
     {
       "title": "Prompting is programming: A query language for large language models",
       "authors": ["Luca Beurer-Kellner", "Marc Fischer", "Martin Vechev"],
       "year": 2023,
       "relevance": "LMQL constrained generation framework, direct comparison baseline."
     },
     {
       "title": "Efficient guided generation for large language models",
       "authors": ["Brandon T. Willard", "Rémi Louf"],
       "year": 2023,
       "relevance": "Outlines framework for constrained generation with pre-computed regex checkers."
     },
     {
       "title": "Synchromesh: Reliable code generation from pre-trained language models",
       "authors": ["Gabriel Poesia"],
       "year": 2022,
       "relevance": "Prior work on minimally invasive constrained decoding for code generation."
     },
     {
       "title": "Grammar-constrained decoding for structured NLP tasks without finetuning",
       "authors": ["Saibo Geng"],
       "year": 2023,
       "relevance": "GCD constrained decoding method, direct comparison."
     },
     {
       "title": "PICARD: Parsing incrementally for constrained auto-regressive decoding from language models",
       "authors": ["Torsten Scholak"],
       "year": 2021,
       "relevance": "Online parser-guided constrained decoding for SQL generation."
     },
     {
       "title": "Accelerating large language model decoding with speculative sampling",
       "authors": ["Charlie Chen"],
       "year": 2023,
       "relevance": "Speculative decoding technique that DOMINO adapts for constrained generation."
     },
     {
       "title": "Training verifiers to solve math word problems",
       "authors": ["Karl Cobbe"],
       "year": 2021,
       "relevance": "GSM8K benchmark used for evaluating constrained decoding accuracy impact."
     }
   ]
 }