ai-research-survey

scan.json (28092B)

---

 {
   "paper": {
     "title": "AdapTrack: Constrained Decoding without Distorting LLM's Output Intent",
     "authors": [
       "Yongmin Li",
       "Jia Li",
       "Ge Li",
       "Zhi Jin"
     ],
     "year": 2025,
     "venue": "arXiv preprint (Conference acronym 'XX' placeholder indicates unpublished/preprint)",
     "arxiv_id": "2510.17376"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.3 states 'We open-source the modified package together with our source code' but NO repository URL, GitHub link, or archive link is provided anywhere in the paper. The schema requires 'a working URL or archive.' A claim of open-source release without a verifiable link is NO."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper's primary novel datasets (TFv1 with 419 APIs, TFv1Real with 1,000 sampled files) are constructed by the authors but no download link or URL is provided. While HumanEval and MBPP are publicly available, the key custom datasets that represent the paper's contribution are not verifiably released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper names libraries (PyTorch, Transformers, transformers-CFG) and hardware (64-core Intel Xeon, 8 NVIDIA RTX A6000 GPUs) but provides no library version numbers, no requirements.txt, no Dockerfile, and no environment setup section with specific dependency versions. This is insufficient to recreate the experimental environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the algorithm and mentions open-sourcing code, but does not provide step-by-step instructions for reproducing the experimental results. No README commands, scripts, or reproduction guide are described in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1, 2, 3, and 4 are point estimates only (e.g., '80.19%', '49.30%'). No confidence intervals, error bars, or standard deviations are reported for any main result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims ('AdapTrack performs significantly better than constrained decoding') but provides no statistical significance tests (no p-values, t-tests, or similar). Comparisons are made by directly comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context (e.g., 'improvement of 360.87% compared to constrained decoding, from 10.98% to 50.60%'), which provides enough context to assess effect magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The TFv1Real dataset uses 1,000 randomly picked files (from 12,883 available), and 20 random samples per problem are collected, but no justification is given for why these sample sizes are sufficient to support the claims. No power analysis is mentioned."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, 3, and 4 report single-point results only. No standard deviation, variance, or confidence intervals are reported across runs. Table 5 reports avg/min/max for LM call counts but not for the performance metrics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares AdapTrack against two direct baselines: unconstrained decoding and constrained decoding (Section 6.1). For RQ4, it also compares against ASAp (Adaptive Sampling with Approximate Expected Futures)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "ASAp (Park et al., 2024) is a recent and directly competing method. The other baselines (unconstrained and constrained decoding) are appropriate ablation-style comparisons, not just old methods. The paper acknowledges ASAp as the closest prior work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2.2 and Figure 7 conduct experiments with limited backtracking distances (0, 1, 2, 4, 8), showing how performance changes as a key component (backtrack distance) is varied. The robustness experiments on temperature and model size also serve as ablation-like analyses."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports EM@k metrics (k=1,3,5,10,20) for API completion tasks and pass@k for general code generation, plus KL divergence for distribution alignment — at least three distinct metric types."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a decoding algorithm evaluated on automated code correctness benchmarks (exact match, pass@k). Human evaluation is not relevant to claims about whether generated code satisfies API constraints."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "TFv1Real was randomly sampled from 12,883 files (Section 6.2), and the paper uses established held-out benchmarks HumanEval and MBPP. The TFv1 dataset is synthetic but constructed specifically for evaluation, not for method tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model (four different code LLMs), by dataset (TFv1 v1 setting, TFv1 v2 setting, TFv1Real, HumanEval, MBPP), and by DSL subdataset (SLIA, INV-BV, CP, binary). Table 4 provides per-dataset KL divergence."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 discusses cases where AdapTrack performs worse than unconstrained decoding (e.g., 'pass@1 of StarCoder2 7B and CodeLlama 7B on MBPP'), attributing it to the constrainer's limitations. Section 6.4 discusses INV-BV where all methods perform similarly."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly notes that AdapTrack sometimes performs worse than unconstrained decoding (Table 3, pass@1 for StarCoder2 7B and CodeLlama 7B on MBPP), and explains the failure case. The outlier in pass@20 of CodeLlama 7B on HumanEval is also noted."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims improvements of up to 360.87% on TFv1, 38.93% on TFv1Real, 7.84% on HumanEval, and 6.42% on MBPP over constrained decoding. These specific numbers are confirmed in Tables 1, 2, and 3. The theoretical proof is presented in Section 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim is that AdapTrack improves performance by avoiding distribution distortion. The paper supports this causally via: (1) a mathematical proof that AdapTrack follows the model's true distribution; (2) controlled experiments where only the decoding method changes while all other variables are held constant; (3) distribution alignment experiments (RQ4) showing lower KL divergence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Constrained Decoding without Distorting LLM's Output Intent' and abstract frame the contribution broadly for 'language model-based code generation.' However, experiments are conducted only on 7B base models (with 0.5B-32B robustness check), only in Python and TypeScript, and only with specific constrainer types. The paper does not explicitly bound its claims to these settings. The schema says 'If the paper tests on Python and claims results for code generation generally, NO.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 discusses two specific threats to validity: (1) whether the problem is specific to deprecated APIs only (mitigated by RQ1 TFv1 v1-setting and RQ3 experiments); (2) whether the problem is model-capability-dependent (mitigated by using multiple models). These are specific alternative explanations, not generic boilerplate."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are named: 'Qwen2.5 Coder 7B', 'DeepSeek Coder Base 6.7B', 'StarCoder2 7B', 'CodeLlama Python 7B', 'Mistral-7B' for RQ4. These are specific enough to identify the models used."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "For the RQ4 DSL experiments, prompts are described as containing 'a grammar and specification' plus '3 in-context examples in the form of (Problem, Solution) pairs', but the actual prompt text is not provided. For other experiments, the code context format is shown (Figure 3) but the full prompt is not given."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is specified as 1 for main experiments (Section 7.1.1). Section 7.1.1 also specifies temperatures tested (0.1 to 1.0). The top-p variant used in RQ3 is described. A timeout of 60 seconds per token validity check is specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "AdapTrack is a decoding algorithm, not an agentic scaffold. There is no agentic scaffolding in this work — the method operates directly at the token generation level without tool use, memory management, or agent loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 describes the full data pipeline for TFv1Real with explicit steps and counts: 46,785 Python files collected, deduplicated and split, files containing tf.compat.v1 in prefix removed, non-API-call suffixes removed → 14,237 files; then filtered by context length >3,584 tokens → 12,883 files; then randomly sampled 1,000 files. Each filtering criterion is explicitly stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 'Threats to validity' discusses limitations, including dataset selection threats (7.3.1) and model selection threats (7.3.2). This is a dedicated subsection with substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3.1 specifically addresses whether the problem is only present in deprecated APIs, and 7.3.2 addresses whether the improvement is model-specific. These are specific threats to this study, not generic disclaimers. However, they lack discussion of efficiency costs in the threats section (addressed elsewhere)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While threats to validity are discussed, the paper does not explicitly state what the results do NOT show. For example, there is no discussion of whether results transfer to instruction-tuned models (only base models are tested), or to languages beyond Python/TypeScript. The discussion of limitations is focused on rebutting threats rather than clearly bounding what was not tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims open-source release but provides no URL or link to datasets in the text. The raw TFv1Real dataset (12,883 filtered GitHub files, then 1,000 sampled) is not independently verifiable from the paper alone."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "TFv1 dataset construction is described in detail (Section 6.1): collected all legal TensorFlow 2.16 APIs, filtered for tf.compat.v1 prefix, excluded APIs with v2 counterparts, retained only lowercase-starting APIs, removed prefix-able APIs, yielding 419 APIs. TFv1Real collection process is similarly detailed (Section 6.2)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This paper has no human participants. Data is collected from GitHub repositories (public data) and official TensorFlow API documentation, not from recruited participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full data pipeline from collection to final analysis is documented for both datasets (Section 6.1 and 6.2) with explicit counts and filtering criteria at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section discloses funding: 'National Key R&D Program under Grant No. 2023YFB4503801, the National Natural Science Foundation of China under Grant No. 62192733, 62192730, 62192731, and the Major Program (JD) of Hubei Province (No.2023BAA024).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the first page: Yongmin Li and Ge Li and Zhi Jin are at Peking University (Key Lab of High Confidence Software Technology); Jia Li is at Tsinghua University College of AI."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from Chinese government research programs (National Key R&D Program, NSFC) which have no financial stake in whether constrained decoding or AdapTrack performs better. Funders are independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. Absence of disclosure is not the same as absence of conflict; the paper does not include a standard competing interests declaration."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses HumanEval and MBPP benchmarks with base LLMs (Qwen2.5 Coder, DeepSeek Coder, StarCoder2, CodeLlama) but does not state the training data cutoff dates for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses HumanEval (2021) and MBPP (2021) with models trained after these benchmarks were published. No discussion of whether these benchmarks appear in the training data is provided."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval and MBPP were both published in 2021. All models used (Qwen2.5, DeepSeek, StarCoder2, CodeLlama) were trained after 2021 and may have seen these benchmarks in training. The paper does not address this contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. It is a benchmark evaluation of a decoding algorithm."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports the number of language model calls per sample (avg, min, max) for AdapTrack vs. baselines, showing AdapTrack requires approximately 20-60% more LM calls. This is a meaningful proxy for inference cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The hardware (8 NVIDIA RTX A6000 GPUs) is specified but total GPU hours, wall-clock time for experiments, or total API cost is not reported. The paper notes efficiency concerns but does not quantify the total compute budget."
       }
     }
   },
   "claims": [
     {
       "claim": "AdapTrack achieves up to 360.87% improvement over constrained decoding on the synthetic TFv1 API completion dataset.",
       "evidence": "Table 1 (Section 6.1): DeepSeek Coder Base 6.7B EM@1 in TensorFlow v2 setting improves from 10.98% (constrained decoding) to 50.60% (AdapTrack), which is a 360.87% relative improvement.",
       "supported": "strong"
     },
     {
       "claim": "AdapTrack achieves up to 38.93% improvement over constrained decoding on the real-world TFv1Real API completion dataset.",
       "evidence": "Table 2 (Section 6.2): StarCoder2 7B EM@1 improves from 29.80% to 41.40%, an improvement of 38.93% relative. Results shown for 4 models.",
       "supported": "strong"
     },
     {
       "claim": "AdapTrack achieves up to 7.84% improvement on HumanEval and 6.42% on MBPP over constrained decoding.",
       "evidence": "Table 3 (Section 6.3): Qwen2.5 Coder 7B pass@1 on HumanEval improves from 64.15% to 69.18% (7.84% relative improvement). MBPP improvement 66.67% to 70.77% (6.15%, not exactly 6.42% — the 6.42% must be from another model/metric combination).",
       "supported": "moderate"
     },
     {
       "claim": "AdapTrack's distribution aligns better with the oracle language model distribution than constrained decoding or ASAp, as measured by KL divergence.",
       "evidence": "Table 4 (Section 6.4): AdapTrack achieves average KL divergence of 9.97 vs. 11.46 for constrained decoding and 11.90 for ASAp. Results shown across 4 DSL datasets using Mistral-7B.",
       "supported": "strong"
     },
     {
       "claim": "AdapTrack is robust to temperature variations (0.0 to 1.0) and model sizes (0.5B to 32B), consistently outperforming constrained decoding.",
       "evidence": "Figures 5 and 6 (Section 7.1): Temperature analysis with CodeLlama and Qwen2.5; model size analysis with Qwen2.5 Coder 0.5B-32B series on TFv1 and TFv1Real.",
       "supported": "strong"
     },
     {
       "claim": "The distribution produced by AdapTrack is mathematically identical to the language model's distribution conditioned on constraints.",
       "evidence": "Section 5 provides Theorems 5.1 and 5.2 with formal proofs. The proof shows that P_Q(s) = (1/Q[epsilon]) * P(s), meaning the algorithm samples proportional to the unconstrained model probability.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "AdapTrack addresses a fundamental flaw in constrained decoding where greedily eliminating invalid tokens distorts the language model's probability distribution, causing it to generate suboptimal valid outputs instead of the intended ones. By incorporating rejection-sampling-based adaptive backtracking, AdapTrack maintains a distribution that is provably proportional to the unconstrained model distribution while satisfying constraints. Empirically, AdapTrack achieves dramatic improvements over naive constrained decoding on API completion tasks (up to 360.87% relative improvement) and more modest improvements on general code generation benchmarks (up to 7.84% on HumanEval), with increased compute cost of approximately 20-60% more LM calls.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "Despite making comparative claims ('AdapTrack performs significantly better') across multiple tables, the paper reports only point estimates with no confidence intervals, standard deviations, or significance tests. The word 'significantly' is used colloquially rather than statistically."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "HumanEval and MBPP were published in 2021. All four models evaluated (Qwen2.5 Coder, DeepSeek Coder, StarCoder2, CodeLlama) were trained after 2021 and likely saw these benchmarks in training. This is particularly important because the baseline (unconstrained decoding) could be inflated by memorization, affecting the relative improvement numbers."
     },
     {
       "flag": "No URL provided for open-source release",
       "detail": "The paper claims to open-source code and the modified constrainer package, but no URL is provided in the paper text. This makes the claimed release unverifiable from the paper alone."
     },
     {
       "flag": "Venue placeholder in header",
       "detail": "The paper header contains 'Conference acronym XX, June 03-05, 2018, Woodstock, NY' — a template placeholder indicating this is an unsubmitted or preprint version. This affects the reliability of the review process."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": [
         "Mark Chen",
         "Jerry Tworek",
         "Heewoo Jun"
       ],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and pass@k metric, used as evaluation benchmark in this paper."
     },
     {
       "title": "Grammar-aligned decoding",
       "authors": [
         "Kanghee Park",
         "Jiayu Wang",
         "Taylor Berg-Kirkpatrick",
         "Nadia Polikarpova",
         "Loris D'Antoni"
       ],
       "year": 2024,
       "relevance": "Introduces ASAp (Adaptive Sampling with Approximate Expected Futures), the main competing method for fixing constrained decoding distribution distortion."
     },
     {
       "title": "Monitor-guided decoding of code lms with static analysis of repository context",
       "authors": [
         "Lakshya A Agrawal",
         "Aditya Kanade",
         "Navin Goyal",
         "Shuvendu Lahiri",
         "Sriram Rajamani"
       ],
       "year": 2023,
       "relevance": "Introduces constrained decoding using static analysis tools, directly related to the problem AdapTrack solves."
     },
     {
       "title": "Copiloting the copilots: Fusing large language models with completion engines for automated program repair",
       "authors": [
         "Yuxiang Wei",
         "Chunqiu Steven Xia",
         "Lingming Zhang"
       ],
       "year": 2023,
       "relevance": "Repilot introduces constrained decoding for general-purpose code generation using IDE language servers, closely related to this work."
     },
     {
       "title": "Type-Constrained Code Generation with Language Models",
       "authors": [
         "Niels Mündler",
         "Jingxuan He",
         "Hao Wang",
         "Koushik Sen",
         "Dawn Song",
         "Martin Vechev"
       ],
       "year": 2025,
       "relevance": "Introduces type-system-based constrained decoding used as the constrainer in RQ3 experiments of AdapTrack."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": [
         "Carlos E Jimenez",
         "John Yang",
         "Alexander Wettig",
         "Shunyu Yao",
         "Kexin Pei",
         "Ofir Press",
         "Karthik R Narasimhan"
       ],
       "year": 2024,
       "relevance": "Major benchmark for evaluating LLM coding capabilities on real software engineering tasks, cited as context for code generation tools."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "authors": [
         "Daya Guo",
         "Qihao Zhu",
         "Dejian Yang"
       ],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "One of the four code LLMs evaluated in the paper; relevant to AI code generation capability."
     },
     {
       "title": "Synchromesh: Reliable Code Generation from Pre-trained Language Models",
       "authors": [
         "Gabriel Poesia",
         "Alex Polozov",
         "Vu Le",
         "Ashish Tiwari",
         "Gustavo Soares",
         "Christopher Meek",
         "Sumit Gulwani"
       ],
       "year": 2022,
       "relevance": "Introduces incremental parsing into constrained code generation, a key baseline method."
     },
     {
       "title": "LLMs Meet Library Evolution: Evaluating Deprecated API Usage in LLM-based Code Completion",
       "authors": [
         "Chong Wang",
         "Kaifeng Huang",
         "Jian Zhang",
         "Yebo Feng",
         "Lyuye Zhang",
         "Yang Liu",
         "Xin Peng"
       ],
       "year": 2025,
       "arxiv_id": "2406.09834",
       "relevance": "Directly evaluates the problem of deprecated API usage in LLM-based code completion, the core motivation for AdapTrack."
     },
     {
       "title": "VersiCode: Towards Version-controllable Code Generation",
       "authors": [
         "Tongtong Wu",
         "Weigang Wu",
         "Xingyu Wang"
       ],
       "year": 2024,
       "arxiv_id": "2406.07411",
       "relevance": "Focuses on version-specific code completion and migration, directly related to the API versioning problem addressed by AdapTrack."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": [
         "Jacob Austin",
         "Augustus Odena",
         "Maxwell Nye",
         "Maarten Bosma"
       ],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark used in evaluation; relevant to LLM code generation capability assessment."
     },
     {
       "title": "ROCODE: Integrating Backtracking Mechanism and Program Analysis in Large Language Models for Code Generation",
       "authors": [
         "Xue Jiang",
         "Yihong Dong",
         "Yongding Tao",
         "Huanyu Liu",
         "Zhi Jin",
         "Ge Li"
       ],
       "year": 2025,
       "relevance": "Related work on backtracking in LLM code generation using static analysis, distinct from AdapTrack's distribution-correction approach."
     },
     {
       "title": "Controllable Generation via Locally Constrained Resampling",
       "authors": [
         "Kareem Ahmed",
         "Kai-Wei Chang",
         "Guy Van den Broeck"
       ],
       "year": 2025,
       "relevance": "Gen-C models constraints with constraint circuits and resamples to fix greedy constrained decoding; directly related competing approach."
     }
   ]
 }