ai-research-survey

scan.json (25261B)

---

 {
   "paper": {
     "title": "Fast Controlled Generation from Language Models with Adaptive Weighted Rejection Sampling",
     "authors": ["Benjamin Lipkin", "Benjamin LeBrun", "Jacob Hoover Vigly", "João Loula", "David R. MacIver", "Li Du", "Jason Eisner", "Ryan Cotterell", "Vikash Mansinghka", "Timothy J. O'Donnell", "Alexander K. Lew", "Tim Vieira"],
     "year": 2025,
     "venue": "COLM 2025",
     "arxiv_id": "2504.05410",
     "doi": "10.48550/arXiv.2504.05410"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "AWRS achieves >50x speedup over token masking for constrained decoding with no accuracy loss, by dynamically allocating computation based on constraint difficulty. AWRS-SMC outperforms or matches state-of-the-art baselines across 5 domains (Text-to-SQL, JSON, goal inference, molecular synthesis, pattern matching) while using fewer particles. Runtime scales with KL divergence between constrained and unconstrained distributions, meaning the method is faster for better base models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Two GitHub repositories provided: https://github.com/genlm/genlm-control (library) and https://github.com/genlm/awrs-colm-2025 (experiment replication)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Uses publicly available benchmarks (Spider, JSONSchemaBench, Planetarium, GDB-17). The pattern matching dataset generation pipeline is described in App. J, and the replication repo is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification found in the paper. Hardware is mentioned (L40S, A100 GPUs) but software dependencies are not specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a replication repository is provided, no step-by-step reproduction instructions are included in the paper itself. The paper notes implementations are 'relatively unoptimized' pure Python but does not provide reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% bootstrapped confidence intervals are reported for all accuracy and runtime results in Table 1 and Table 3."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No formal significance tests are used. Comparisons between methods rely on comparing bootstrapped confidence intervals, but no explicit hypothesis tests (p-values, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with absolute accuracy values and runtime in seconds with baselines for context (e.g., ARS-LCD 0.980 vs TM-LCD 0.978, >50x speedup). The reader can compute effect sizes from the provided numbers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes is provided. The number of examples per benchmark is not discussed in terms of statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "95% bootstrapped confidence intervals are reported for all main results, providing spread information. Simulation results in App. F also show variance across Monte Carlo iterations."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five baselines compared: Base LM, TM-LCD (token masking), ARS-LCD, Sample-Verify, and Twisted SMC. Each is clearly described in §4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent work: Twisted SMC (Loula et al., 2025), Sample-Verify approaches from 2024, and the standard TM-LCD approach. Related work in §6 discusses very recent concurrent work (Botta et al., 2025; Mündler et al., 2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The comparison between ARS-LCD (unweighted) and AWRS-SMC (weighted with importance correction) serves as an ablation of the weighting component. The paper also varies number of particles and model sizes (Fig. L.1, Tables 1 and 3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two primary metrics reported: task-specific accuracy and runtime (seconds per example). Each domain also has its own accuracy metric (execution accuracy, QED, pattern adherence, etc.)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant; the tasks have objective ground-truth metrics (execution accuracy, schema validation, pattern matching, PDDL equivalence, QED)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmark splits used: Spider development split, JSONSchemaBench validation splits, Planetarium Blocksworld tasks. These are established held-out evaluation sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by domain (5 domains in Table 1), by model size (Table 3, 1B/8B/70B), and by JSON difficulty level (Github-trivial, -easy, -medium mentioned in §4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where LCD fails (dead-end paths, §2 example with 'mortg'), and Figure 2 shows how AWRS handles hard cases. The paper notes Pattern Matching suffers less from greediness because its local constraint is more precise."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "AWRS-SMC does not always improve over ARS-LCD (Pattern Matching domain, where the local constraint is precise enough). The paper also notes TM-LCD was computationally infeasible for most benchmarks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of orders-of-magnitude fewer constraint evaluations, low-variance unbiased Z estimates, and superiority across 5 benchmarks are all supported by Table 1, Figure 2, and the theoretical analysis."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'AWRS improves runtime' are justified through controlled single-variable comparisons (same model, same benchmark, different decoding algorithm). The ablation between ARS-LCD and AWRS-SMC isolates the importance weighting contribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to the tested domains. The paper states results for specific benchmarks and models (Llama 3.1 8B, etc.). The abstract says 'through extensive empirical evaluation in text-to-SQL, molecular synthesis, goal inference, pattern matching, and JSON domains' rather than claiming general superiority."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses why AWRS-SMC's advantage varies across domains (Pattern Matching has a more precise local constraint, §5). It also discusses the relationship between model quality and runtime, providing both theoretical (§3, App. G) and empirical (Fig. 2) analyses of what drives performance."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures task-specific accuracy (execution accuracy, QED, pattern adherence) and clearly states what each metric measures. It does not overframe these as broader claims beyond the specific tasks."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions stated: Llama 3.1 8B-Instruct, Llama 3.1 8B, Llama 3.2 1B, Llama 3.3 70B. These are specific enough model identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "For molecular synthesis, 'few-shot prompts created by repeatedly selecting 20 random samples from the GDB-17 database' but actual prompt text is not shown. No prompts are provided in the paper or appendix for any domain."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "App. K reports temperature (1.0), max tokens per domain (32-350), ESS thresholds for resampling, particle counts (M=5 for AWRS-SMC, M=10 for baselines), and resampling strategies."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method is a decoding algorithm applied directly to language models."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data sources and processing are described: Spider dev split, JSONSchemaBench validation splits with difficulty tiers, Planetarium Blocksworld with up to 10 objects, GDB-17 database for molecular synthesis, and a detailed pattern generation pipeline in App. J."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. The paper discusses some limitations implicitly (e.g., noting implementations are unoptimized, TM-LCD is infeasible for most benchmarks) but lacks a structured limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential issues like the choice of benchmarks, the impact of constraint checker implementation quality, or generalization to other model families."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly notes it focuses on 'runtime control' (footnote 1), distinguishing from training-based methods. It also notes implementations are 'pure Python and relatively unoptimized' (footnote 5), bounding runtime claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "A replication repository is provided (https://github.com/genlm/awrs-colm-2025) containing 'source code and data to replicate this paper's experiments.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Each benchmark's data source is clearly described: Spider dev split (Yu et al., 2018), JSONSchemaBench validation splits (Geng et al., 2025), Planetarium Blocksworld (Zuo et al., 2024), GDB-17 (Ruddigkeit et al., 2012), and custom pattern matching pipeline (App. J)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pattern matching generation pipeline is detailed in App. J with filtering stages and counts (1503 candidates → 402 after dedup, library compatibility, FSM exclusion, and prefix checks). Other benchmarks use standard splits."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section discloses NSF Graduate Research Fellowship (Grant No. 2141064), NSF SBE Postdoctoral Research Fellowship (Grant No. SMA-2404644), and Mila compute resources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: MIT, ETH Zürich, McGill, Canada CIFAR AI Chair, Mila, Johns Hopkins, Yale, CHI FRO."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from NSF and academic compute (Mila). These are independent of the outcome — no commercial entity with a stake in the results is funding the work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama models on benchmarks like Spider but does not state the training data cutoff dates for the models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Spider or other benchmark examples appeared in Llama's training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Spider was published in 2018, well before Llama's training data. No discussion of contamination risk despite using benchmarks that predate the models."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Runtime in seconds per example is reported for all methods and benchmarks in Table 1 and Table 3. Constraint evaluation costs per domain are in Table 2 (App. I.1)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (L40S, A100 GPUs) but total compute budget (GPU hours, total experiment time) is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. Results are reported with bootstrapped CIs but no mention of varying random seeds across runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. Bootstrapped CIs are computed but it is unclear whether results come from single or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. Choices of particle counts (M=5 for AWRS-SMC, M=10 for baselines) and ESS thresholds appear tuned but no search process is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of M=5 for AWRS-SMC vs M=10 for baselines is not justified through a validation procedure. Different ESS thresholds and resampling strategies are used per domain without explaining selection."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines themselves. No discussion of self-comparison bias or whether their implementations of Sample-Verify and Twisted SMC are faithful."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure L.1 shows accuracy vs runtime tradeoff across different particle counts and model sizes. Table 1 reports both accuracy and runtime for all methods, enabling compute-matched comparisons."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the chosen benchmarks adequately represent real-world constrained generation needs. The paper does not question the construct validity of any benchmark."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding involved. The methods are decoding algorithms applied directly to LMs."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Spider (2018) and other benchmarks predate Llama training, creating contamination risk that is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The constraint checker provides information during generation that may not be available in unconstrained settings, but this is inherent to the method rather than a leakage concern."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and test data share structural similarities."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used or discussed."
       }
     }
   },
   "claims": [
     {
       "claim": "ARS-LCD is >50x faster than token masking (TM-LCD) with no loss of accuracy on constrained decoding.",
       "evidence": "Table 1e: Pattern Matching — ARS-LCD accuracy 0.980 vs TM-LCD 0.978, runtime 0.16s vs 6.91s (§5).",
       "supported": "strong"
     },
     {
       "claim": "AWRS-SMC outperforms or matches all baselines across 5 domains while using fewer particles (M=5 vs M=10).",
       "evidence": "Table 1: AWRS-SMC matches or beats Sample-Verify and Twisted SMC in accuracy across all 5 domains (§5).",
       "supported": "strong"
     },
     {
       "claim": "AWRS runtime scales with the KL divergence between constrained and unconstrained distributions, making it faster for better models.",
       "evidence": "Figure 2 shows empirical correlation between DKL and number of constraint evaluations. Theoretical analysis in Proposition 4 and App. G.2.",
       "supported": "strong"
     },
     {
       "claim": "AWRS-SMC with a smaller LM (1B) outperforms Twisted SMC with larger LMs (8B, 70B) in accuracy and runtime.",
       "evidence": "Figure L.1 and Table 3: AWRS-SMC with Llama 3.2 1B (0.974 accuracy) beats Twisted SMC with Llama 3.1 8B (0.796) and Llama 3.3 70B (0.846) on Pattern Matching.",
       "supported": "strong"
     },
     {
       "claim": "The Z estimator produced by AWRS is unbiased.",
       "evidence": "Formal proof in Proposition 3 / App. D using the RAVI framework. Empirical validation in Figure F.1 showing MAE trending to 0 with increasing Monte Carlo samples.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Asymmetric particle counts",
       "detail": "AWRS-SMC uses M=5 particles while Sample-Verify and Twisted SMC use M=10. While the paper acknowledges this ('with half the number of particles'), the comparison is not fully controlled — giving AWRS-SMC the same budget might show different results."
     },
     {
       "flag": "TM-LCD baseline run on only one benchmark",
       "detail": "The key speedup claim (>50x) is demonstrated on only the Pattern Matching domain. The paper states TM-LCD was 'computationally infeasible' for other benchmarks but does not provide partial results or projected costs."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Spider (2018) and other benchmarks predate Llama model training, creating significant contamination risk. Base LM accuracy on Spider (52.3%) is high enough that memorization could be a factor, and this is never discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "Synchromesh: Reliable code generation from pre-trained language models",
       "authors": ["Gabriel Poesia", "Alex Polozov", "Vu Le", "Ashish Tiwari", "Gustavo Soares", "Christopher Meek", "Sumit Gulwani"],
       "year": 2022,
       "relevance": "Key prior work on constrained decoding for code generation from LMs."
     },
     {
       "title": "SynCode: Improving LLM code generation with grammar augmentation",
       "authors": ["Shubham Ugare", "Tarun Suresh", "Hangoo Kang", "Sasa Misailovic", "Gagandeep Singh"],
       "year": 2024,
       "arxiv_id": "2403.01632",
       "relevance": "Grammar-based constrained decoding for LLM code generation."
     },
     {
       "title": "Efficient guided generation for large language models",
       "authors": ["Brandon T Willard", "Rémi Louf"],
       "year": 2023,
       "arxiv_id": "2307.09702",
       "relevance": "Outlines library for structured LLM generation with grammar constraints."
     },
     {
       "title": "XGrammar: Flexible and efficient structured generation engine for large language models",
       "authors": ["Yixin Dong", "Charlie F Ruan", "Yaxing Cai", "Ruihang Lai", "Ziyi Xu", "Yilong Zhao", "Tianqi Chen"],
       "year": 2024,
       "arxiv_id": "2411.15100",
       "relevance": "State-of-the-art structured generation engine with optimized grammar-based constrained decoding."
     },
     {
       "title": "Syntactic and semantic control of large language models via sequential Monte Carlo",
       "authors": ["João Loula", "Benjamin LeBrun", "Li Du", "Ben Lipkin"],
       "year": 2025,
       "relevance": "Prior work by same group on SMC-based constrained LLM generation, used as baseline."
     },
     {
       "title": "CRANE: Reasoning with constrained LLM generation",
       "authors": ["Debangshu Banerjee", "Tarun Suresh", "Shubham Ugare", "Sasa Misailovic", "Gagandeep Singh"],
       "year": 2025,
       "arxiv_id": "2502.09061",
       "relevance": "Recent work on constrained LLM generation with reasoning capabilities."
     },
     {
       "title": "Grammar-aligned decoding",
       "authors": ["Kanghee Park", "Jiayu Wang", "Taylor Berg-Kirkpatrick", "Nadia Polikarpova", "Loris D'Antoni"],
       "year": 2024,
       "relevance": "Addresses distortion in grammar-constrained decoding, related to LCD greediness problem."
     },
     {
       "title": "PICARD: Parsing incrementally for constrained auto-regressive decoding from language models",
       "authors": ["Torsten Scholak", "Nathan Schucher", "Dzmitry Bahdanau"],
       "year": 2021,
       "relevance": "Influential work on incremental constrained decoding for text-to-SQL generation."
     },
     {
       "title": "IterGen: Iterative structured LLM generation",
       "authors": ["Shubham Ugare", "Rohan Gumaste", "Tarun Suresh", "Gagandeep Singh", "Sasa Misailovic"],
       "year": 2025,
       "relevance": "Alternative approach to structured generation allowing backtracking on constraint violations."
     },
     {
       "title": "Generating structured outputs from language models: Benchmark and studies",
       "authors": ["Saibo Geng", "Hudson Cooper", "Michai Moskal"],
       "year": 2025,
       "arxiv_id": "2501.10868",
       "relevance": "JSONSchemaBench benchmark used in this paper's evaluation of structured output generation."
     }
   ]
 }