ai-research-survey

calibration.json (17848B)

---

 {
   "paper_slug": "accelerating-large-language-2023",
   "total_questions": 50,
   "agreement_count": 49,
   "disagreement_count": 1,
   "agreement_rate": 0.98,
   "disagreements": [
     {
       "category": "statistical_methodology",
       "question": "variance_reported",
       "sonnet_applies": true,
       "sonnet_answer": true,
       "opus_applies": true,
       "opus_answer": false,
       "opus_justification": "Figure 1 shows standard deviation for mean sampling time, but this appears to be within-run variation across different sequences in a single experimental run, not variance across independent experimental runs or seeds. The paper never mentions repeating experiments across multiple runs or seeds. The schema asks for 'variance or standard deviation reported across experimental runs' and states 'If the paper reports single-run numbers only, NO.' The benchmark quality metrics (ROUGE-2, HumanEval pass rate) are reported as single point estimates with no variance at all.",
       "sonnet_justification": "Figure 1 shows standard deviation for the mean sampling time metric. The paper notes 'The time taken per SpS/ArS loop has low variance' and the figure caption confirms 'with standard deviation' is displayed.",
       "direction": "sonnet_generous"
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link, Zenodo archive, or code repository URL is provided anywhere in the paper. The algorithm is described in pseudocode (Algorithms 1 and 2) but no source code is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: XSum (Narayan et al., 2018) and HumanEval (Chen et al., 2021). Per schema, 'If the data is a standard public benchmark they didn't modify, YES.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '16 TPU v4s' and 'Megatron style' sharding but provides no software environment specification (no requirements.txt, Dockerfile, conda environment, or library versions). Hardware alone is insufficient per the schema."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided. Reproduction requires access to proprietary DeepMind infrastructure, Chinchilla weights, and a custom-trained draft model."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 (left panel) shows standard deviation bars for mean sampling time across different K values. The caption explicitly states 'with standard deviation.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Speedup claims and benchmark score comparisons (e.g., ROUGE-2: 0.112 vs 0.114, HumanEval: 45.1% vs 47.0%) are made without p-values, t-tests, or any formal testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Speedup factors are clearly reported with baseline context: 1.92x, 2.01x, and 2.46x speedups relative to ArS at 14.1ms/token (Table 1). The magnitude of improvement is fully quantified."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 11,305 sequences for XSum and 16,400 samples for HumanEval but provides no justification for these specific sizes and no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Figure 1 shows standard deviation for mean sampling time, but this is within-run variation across different sequences in a single experimental run, not variance across independent experimental runs or seeds. The paper never mentions repeating experiments across multiple runs. The benchmark quality metrics (ROUGE-2, HumanEval pass rate) are reported as single point estimates with no spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Standard auto-regressive sampling (ArS) is the baseline throughout, with direct comparisons in Table 1 for both XSum and HumanEval benchmarks."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Standard auto-regressive sampling is the natural and current baseline for decoding acceleration work. The paper also discusses concurrent work by Leviathan et al. (2022) in related work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 provides a systematic exploration of the K (lookahead) hyperparameter from 0 to 7, showing how speedup, acceptance rate, and loop time vary. The paper also discusses why a standard 7B model fails as a draft model in distributed settings, comparing draft model choices."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports both quality metrics (ROUGE-2 for XSum, pass rate for HumanEval) and latency metrics (ms/token, speedup factor). Figure 1 additionally reports acceptance rate and total loop time."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper's claims concern decoding speed and distributional equivalence, both objectively verifiable through timing measurements and benchmark metrics. Human evaluation is irrelevant to these claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "XSum and HumanEval are standard test benchmarks with defined test splits. No model training or fine-tuning was performed on these benchmarks; they are used purely for evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task (XSum vs HumanEval), by decoding method (nucleus vs greedy for XSum), and Figure 1 shows separate curves per domain across K values."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses several cases where the approach underperforms: XSum with nucleus at larger K values experiences latency regression (optimal at K=3), larger K increases variance problematic for P90/P99 latencies, and a standard 7B model fails as an efficient draft in distributed settings."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that a chinchilla-optimal 7B model 'would provide only a modest speedup' and that serving 7B on 16 TPUs 'actually increases the latency.' The speedup plateauing/regression at larger K is also a negative finding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '2-2.5x decoding speedup' — Table 1 shows 1.92x-2.46x. 'Without compromising sample quality' — benchmark scores match within noise (ROUGE-2 0.112 vs 0.114, HumanEval 45.1% vs 47.0%). Both claims are well supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claim (SpS causes speedup) is supported by controlled single-variable comparisons (SpS vs ArS on identical model, hardware, and benchmarks). The distributional equivalence claim is formally proven (Theorem 1). The K ablation is a controlled manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Results are only on Chinchilla 70B with a specific 4B draft model on 16 TPU v4s, but the title says 'Accelerating Large Language Model Decoding' and the conclusion says the method 'scales well with the appropriate draft model and complements many existing techniques' without bounding these claims to the tested setup."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for results are discussed. The hypothesis for HumanEval's higher speedup (code has common sub-sequences) is stated without testing or considering alternatives. No threats-to-validity section exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "'Chinchilla' is named without a specific version, checkpoint, or snapshot date. The draft model is described by architecture hyperparameters (Table 2) but has no version identifier. Per schema, naming a model without version/snapshot is NO."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '1-shot prompt' for XSum and '100-shot' for HumanEval but the actual prompt text is never provided. Per schema, describing prompts without providing actual text is NO."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides draft model architecture hyperparameters (d_model=6144, heads=48, layers=8, params=4B). Table 1 specifies sampling parameters: nucleus p=0.8 for XSum, p=0.95 and temperature=0.8 for HumanEval. K=4 for main results."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a standalone sampling algorithm for transformer decoding, not an agent-based system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data preprocessing steps are documented. For XSum, the 1-shot prompt construction is not described. For HumanEval, the 100-shot setup details are not provided. Maximum sequence lengths are stated (128, 512) but no preprocessing pipeline is described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. The conclusion briefly mentions the method works 'in the small batch size setting' but this is a single clause, not substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats-to-validity are discussed. The paper does not address specific concerns such as dependence on draft model quality, hardware specificity, limited benchmark coverage, or batch size constraints."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what results do not show. No statements about what populations/settings/hardware are excluded from the claims. The 'small batch size setting' mention in the conclusion is the only scope qualifier but is not developed."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (timing logs, TPU profiles, per-sequence benchmark outputs) is made available. Only aggregated results (means, speedup ratios, benchmark scores) are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes timing data collection: 'The time taken per SpS/ArS loop has low variance, and we can measure it directly from TPU profiles. To obtain the average speedup, standard deviations and other metrics, we log the amount of tokens generated for each speculative loop.' Benchmark parameters (11,305 XSum sequences, 16,400 HumanEval samples) are stated."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a systems paper using standard benchmarks. The data source is standard public benchmarks (XSum, HumanEval)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from benchmark inputs to final aggregated results is not fully documented. How benchmark data was prepared, how the 1-shot/100-shot prompts were constructed, and how raw timing data was aggregated into the reported metrics are not explained step by step."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The acknowledgments section thanks DeepMind colleagues but lists no grants, sponsors, or funding agencies. The work was clearly funded by DeepMind/Alphabet but this is not stated."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All six authors are clearly identified as being from DeepMind ('All authors from DeepMind'). They evaluate Chinchilla, a DeepMind model, and the institutional affiliation is transparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "DeepMind employs all authors and owns Chinchilla. DeepMind/Alphabet has a direct commercial interest in demonstrating faster inference for its models. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper. Per schema, absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff for Chinchilla is not stated. While the paper's primary claim is about decoding speed rather than model capability, it does report HumanEval and XSum benchmark scores, and the training cutoff is relevant for assessing whether these scores are meaningful for the distributional equivalence verification."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval or XSum examples appeared in Chinchilla's training data. Both benchmarks predate Chinchilla's training and could plausibly be contaminated."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (published 2021) and XSum (published 2018) were both available before Chinchilla's training. The paper does not address contamination risk, though contamination would affect both ArS and SpS equally, making it less critical for the speedup claim."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a systems paper evaluating a decoding algorithm on standard benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a systems paper evaluating a decoding algorithm on standard benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a systems paper evaluating a decoding algorithm on standard benchmarks."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a systems paper evaluating a decoding algorithm on standard benchmarks."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a systems paper evaluating a decoding algorithm on standard benchmarks."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a systems paper evaluating a decoding algorithm on standard benchmarks."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a systems paper evaluating a decoding algorithm on standard benchmarks."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference latency is the central metric of the paper. Table 1 reports mean token time in ms/token for both ArS and SpS. Figure 1 shows mean sampling time for 128 tokens across different K values."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states the draft model was trained on '16 TPU v4s' but does not specify total training time, TPU hours, FLOPs, or total compute cost. The computational budget for training the draft model or running the experiments is not quantified."
       }
     }
   }
 }