ai-research-survey

calibration.json (19981B)

---

 {
   "paper_slug": "adaptrack-constrained-decoding-2025",
   "calibration_date": "2026-02-28",
   "calibration_model": "opus",
   "scan_model": "sonnet",
   "total_questions": 50,
   "agreement_count": 46,
   "disagreement_count": 4,
   "agreement_rate": 0.92,
   "disagreements": [
     {
       "category": "artifacts",
       "question": "code_released",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "The paper states 'We open-source the modified package together with our source code' (Section 6.3) but provides NO repository URL or archive link anywhere in the paper text. The schema explicitly states: 'Only YES if a working URL or archive is provided.' A promise or claim of release without a verifiable link does not meet this criterion."
     },
     {
       "category": "artifacts",
       "question": "data_released",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "While HumanEval and MBPP are publicly available benchmarks, the paper's primary novel datasets (TFv1 and TFv1Real) are constructed by the authors and not released with any download link or URL. The open-source claim has no accompanying URL. The schema requires actual availability, not claims of future release alongside code."
     },
     {
       "category": "artifacts",
       "question": "environment_specified",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "The paper names libraries (PyTorch, Transformers, transformers-CFG) and hardware (64-core Intel Xeon, 8 NVIDIA RTX A6000 GPUs) but provides NO library version numbers, no requirements.txt, no Dockerfile, and no detailed environment setup section. The schema says 'Mentioning Python 3.x alone is NOT enough — there must be enough detail to recreate the environment.' Similarly, naming libraries without versions is insufficient to recreate the environment."
     },
     {
       "category": "claims_and_evidence",
       "question": "generalization_bounded",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "The paper's title 'Constrained Decoding without Distorting LLM's Output Intent' and abstract discuss 'language model-based code generation' broadly, but experiments are only on 7B base models in Python and TypeScript. The schema says: 'If the paper tests on Python and claims results for code generation generally, NO.' The threats to validity section (7.3) addresses some concerns but does not explicitly bound generalization to the tested settings (base models only, Python/TypeScript only, 7B-scale primarily). Instruction-tuned models, other languages, and other constraint types are not discussed as scope boundaries."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.3 states 'We open-source the modified package together with our source code' but NO repository URL, GitHub link, or archive link is provided anywhere in the paper. The schema requires 'a working URL or archive.' A claim of open-source release without a verifiable link is NO."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper's primary novel datasets (TFv1 with 419 APIs, TFv1Real with 1,000 sampled files) are constructed by the authors but no download link or URL is provided. While HumanEval and MBPP are publicly available, the key custom datasets that represent the paper's contribution are not verifiably released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper names libraries (PyTorch, Transformers, transformers-CFG) and hardware (64-core Intel Xeon, 8 NVIDIA RTX A6000 GPUs) but provides no library version numbers, no requirements.txt, no Dockerfile, and no environment setup section with specific dependency versions. This is insufficient to recreate the experimental environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README with commands, or reproduction scripts are described in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-4 are point estimates only. No confidence intervals, error bars, or standard deviations are reported for any performance metric."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'AdapTrack performs significantly better') but provides no statistical significance tests — no p-values, t-tests, or similar. All comparisons are raw point estimate comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., 'improvement of 360.87% compared to constrained decoding' with absolute numbers (from 10.98% to 50.60%). This provides sufficient context to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes: 419 APIs in TFv1, 1,000 files in TFv1Real (from 12,883 available), 20 random samples per problem, 2000 samples for DSL. No power analysis or reasoning for these choices."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All performance tables (1-4) report single-point results with no standard deviation, variance, or spread measures across runs. Table 5 reports avg/min/max for LM call counts but not for the main performance metrics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares AdapTrack against unconstrained decoding, constrained decoding (Sections 6.1-6.3), and ASAp (Section 6.4). Multiple appropriate baselines are included."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "ASAp (Park et al., 2024) is a recent competing method. Unconstrained and constrained decoding are appropriate fundamental baselines for this problem. The paper acknowledges ASAp as the closest prior work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2.2 and Figure 7 conduct experiments with limited backtracking distances (0, 1, 2, 4, 8), isolating the key component's contribution. Robustness experiments on temperature (Section 7.1.1) and model size (Section 7.1.2) provide additional component analysis."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses EM@k (exact match at k=1,3,5,10,20), pass@k for general code generation, and KL divergence for distribution alignment — at least three distinct metric types."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a decoding algorithm evaluated on automated code correctness metrics (exact match, pass@k, KL divergence). Human evaluation is not relevant to claims about distribution alignment and constraint compliance."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "HumanEval and MBPP are standard held-out benchmarks. TFv1 is a synthetic evaluation dataset not used for tuning. TFv1Real was randomly sampled from 12,883 files for evaluation only. No evidence of data leakage between tuning and evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model (4 code LLMs), by dataset (TFv1 v1/v2, TFv1Real, HumanEval, MBPP), and by DSL subdataset (SLIA, INV-BV, CP, binary in Table 4). Individual model performance is visible in all tables."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 discusses cases where AdapTrack performs worse than unconstrained decoding (pass@1 for StarCoder2 7B and CodeLlama 7B on MBPP), attributing it to the constrainer's limitation. Section 6.4 discusses INV-BV where all methods perform similarly."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly reports that AdapTrack sometimes performs worse than unconstrained decoding (Table 3: pass@1 for StarCoder2 7B and CodeLlama 7B on MBPP), and notes the outlier in pass@20 of CodeLlama 7B on HumanEval."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 360.87% improvement on TFv1, 38.93% on TFv1Real, 7.84% on HumanEval, and 6.42% on MBPP are confirmed in Tables 1, 2, and 3. The theoretical proof is in Section 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claim that AdapTrack improves performance by avoiding distribution distortion is supported by: (1) mathematical proof in Section 5, (2) controlled experiments where only the decoding method varies, (3) KL divergence experiments (RQ4) demonstrating distribution alignment. These constitute adequate causal evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Constrained Decoding without Distorting LLM's Output Intent' and abstract frame the contribution broadly for 'language model-based code generation.' However, experiments are conducted only on 7B base models (with 0.5B-32B robustness check), only in Python and TypeScript, and only with specific constrainer types. The paper does not explicitly bound its claims to these settings. The schema says 'If the paper tests on Python and claims results for code generation generally, NO.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 discusses two specific alternative explanations: (1) whether the problem only manifests with deprecated APIs (7.3.1), mitigated by TFv1 v1-setting and RQ3 experiments; (2) whether the problem is model-capability-dependent (7.3.2), mitigated by multi-model experiments. These are specific to this study."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names are given: Qwen2.5 Coder 7B, DeepSeek Coder Base 6.7B, StarCoder2 7B, CodeLlama Python 7B, Mistral-7B. These are open-weight models whose names uniquely identify specific model checkpoints on HuggingFace, unlike API-based models that change over time."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figure 3 shows an example code context format for TFv1, but actual prompts for all problems are not provided. For RQ4, prompts contain 'a grammar and specification' with '3 in-context examples' but the actual prompt text is not shown. HumanEval/MBPP use standard prompts from MultiPL-E but the paper does not include them."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature specified as 1 for main experiments (Section 7.1.1). Temperature range tested (0.1-1.0). Top-p variant for RQ3 described. 60-second timeout per token validity check specified. 20 samples per problem for EM@k. 2000 samples for DSL experiments."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "AdapTrack is a decoding algorithm operating at the token generation level. There is no agentic scaffolding (no tool use, memory management, or agent loops)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 describes TFv1 construction: collected all TensorFlow 2.16 APIs, filtered for tf.compat.v1 prefix, excluded APIs with v2 counterparts, retained only lowercase-starting APIs, removed prefix-able APIs, yielding 419 APIs. Section 6.2 describes TFv1Real: 46,785 files → deduplicated/split → removed prefix files → removed non-API suffixes → 14,237 → filtered >3,584 tokens → 12,883 → randomly sampled 1,000."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 'Threats to validity' provides a dedicated subsection with two specific subsections (7.3.1 Dataset selection, 7.3.2 Model selection) containing substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3.1 specifically addresses whether the distortion problem only occurs with deprecated APIs. Section 7.3.2 addresses whether the improvement is model-capability-specific. Both are specific to this study and mitigated with additional experiments."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No discussion of whether results transfer to instruction-tuned models (only base models tested), to languages beyond Python/TypeScript, to other constraint types, or to production settings. The threats section focuses on rebutting concerns rather than explicitly bounding scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims open-source release but provides no URL. Neither TFv1 nor TFv1Real datasets are available for independent verification from the paper alone. Raw experimental outputs are not provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "TFv1 construction: collected legal TensorFlow 2.16 APIs from official website, applied explicit filtering criteria, yielding 419 APIs (Section 6.1). TFv1Real: collected 46,785 Python files from GitHub using tf.compat.v1 API search, applied documented filtering pipeline (Section 6.2)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sourced from GitHub repositories (public data) and official TensorFlow API documentation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented for both datasets with explicit counts at each stage: TFv1 (all TF 2.16 APIs → filter compat.v1 → exclude v2 counterparts → lowercase filter → prefix filter → 419). TFv1Real (46,785 → dedup/split → 14,237 → 12,883 → 1,000)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists: National Key R&D Program (2023YFB4503801), NSFC grants (62192733, 62192730, 62192731), and Hubei Province Major Program (2023BAA024)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Yongmin Li, Ge Li, and Zhi Jin at Peking University (Key Lab of High Confidence Software Technology); Jia Li at Tsinghua University (College of AI)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding from Chinese government research programs (NSFC, National Key R&D Program, Hubei Province) which have no financial stake in whether AdapTrack outperforms constrained decoding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present anywhere in the paper. The schema states: 'If there is no competing interests statement at all, NO — absence of disclosure is not the same as absence of conflict.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses HumanEval and MBPP benchmarks with LLMs (Qwen2.5 Coder, DeepSeek Coder, StarCoder2, CodeLlama, Mistral) but does not state the training data cutoff date for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) are used with models trained after these benchmarks were published. No discussion of potential overlap between training data and benchmark examples."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval and MBPP were both published in 2021. All models used were trained well after 2021 and may have seen these benchmarks in training. The paper does not address this contamination risk, which could inflate baseline numbers and affect relative improvement measurements."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of a decoding algorithm."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. IRB approval not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports the number of LM calls per sample (avg, min, max) for all methods across all models, showing AdapTrack requires approximately 20-60% more LM calls. This is a meaningful proxy for inference cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (8 NVIDIA RTX A6000 GPUs, 64-core Intel Xeon) but total GPU hours, wall-clock time for all experiments, or total compute budget is not quantified."
       }
     }
   }
 }