ai-research-survey

calibration.json (17916B)

---

 {
   "paper_slug": "adafuse-adaptive-ensemble-2026",
   "total_questions": 50,
   "agreement_count": 48,
   "disagreement_count": 2,
   "agreement_rate": 0.96,
   "disagreements": [
     {
       "category": "evaluation_design",
       "question": "human_evaluation",
       "sonnet_applies": false,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": false,
       "opus_justification": "The paper includes machine translation evaluation (FLORES En-De, De-En) where BLEU scores are known to correlate imperfectly with human translation quality judgments. Human evaluation is relevant for assessing translation output quality and is commonly used in MT research. The paper did not include any human evaluation. For the QA and math tasks, automated metrics (exact match, accuracy) are sufficient, but the inclusion of translation tasks makes human evaluation applicable.",
       "sonnet_justification": "The paper evaluates an automated decoding algorithm on NLP benchmarks using automated metrics (exact match, BLEU). Human evaluation is not relevant since ground-truth answers exist for all benchmarks used.",
       "direction": "applies_boundary"
     },
     {
       "category": "claims_and_evidence",
       "question": "generalization_bounded",
       "sonnet_applies": true,
       "sonnet_answer": true,
       "opus_applies": true,
       "opus_answer": false,
       "opus_justification": "The paper's title 'AdaFuse: Adaptive Ensemble Decoding with Test-Time Scaling for LLMs' suggests general applicability to LLMs. The abstract claims ADAFUSE 'consistently outperforms strong ensemble baselines' without bounding this to the tested setting. Results cover only 4 open-source models of ~7-8B parameters on 6 benchmarks across 3 task types. While the Limitations section notes the method requires open-source models with token-level likelihoods, it does not explicitly bound generalization claims to the tested model sizes (~8B), tested task types (QA, math, translation), or the number of languages tested. The conclusion speaks broadly of 'diverse tasks and evolving generation contexts.' Per the schema, a broad title with narrow evaluation warrants answer=false.",
       "sonnet_justification": "The Limitations section explicitly states that ADAFUSE 'is currently limited to open or fully accessible models' that provide token-level likelihoods. The paper evaluates on 6 benchmarks across 3 task types with 4 specific open-source models, and the claims are bounded to this setting.",
       "direction": "sonnet_generous"
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'The code is available at https://github.com/CCM0111/AdaFuse.' A working GitHub URL is provided directly in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All benchmarks used (NaturalQuestions, SQuAD, TriviaQA, GSM8K, FLORES) are publicly available standard datasets. Appendix B confirms they use standard datasets 'without modifying their contents.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.4 specifies the full software environment: PyTorch v2.4.1 (CUDA 12.1), HuggingFace Transformers v4.51.3, Tokenizers v0.21.0, Accelerate v1.6.0, and Datasets v3.0.2. This is sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper or appendix. Appendix B.5 describes the repository as having 'modular components' with 'inline comments' but does not provide specific commands or steps to reproduce experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for any results. Table 1 reports only point estimates. Appendix D.3 states 'we report results from a deterministic decoding setting,' indicating single-run point estimates."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims of outperformance (e.g., 'consistently outperforms') are based solely on comparing point estimates without any p-values, t-tests, or other hypothesis tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports both absolute scores and relative percentage improvements (e.g., '+10.01%' on NQ, '+4.12%' on SQuAD) with baseline scores provided, enabling effect size interpretation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix C lists dataset sizes (e.g., 3610 NQ, 2500 SQuAD, 1319 GSM8K) but no justification is given for these sizes. No power analysis or discussion of statistical adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Appendix D.3 explicitly states results are from a 'deterministic decoding setting.' No standard deviation, variance, or spread across multiple runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four ensemble baselines are compared: LLM-BLENDER (sample-level), DEEPEN (token-level), SWEETSPAN (span-level), and UNITE (token-level), covering all major ensemble granularities."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baselines are recent: LLM-BLENDER (2023), DEEPEN (2024), SWEETSPAN (2024), UNITE (2024). These represent the current state of the art in LLM ensemble methods."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 ablates adaptive vs. fixed-length word commitment (lengths 1, 2, 3). Section 4.4 ablates the diversity-aware ensemble scaling component. Appendix F compares diversity-aware scaling vs. beam-search scaling."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Exact match accuracy for QA tasks, answer accuracy for GSM8K, spBLEU for machine translation, and BERTScore as an additional semantic similarity metric (Appendix D.4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper includes machine translation evaluation where human evaluation is relevant (BLEU correlates imperfectly with human quality judgments). No human evaluation was conducted; all evaluation is automated."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard held-out test splits of established benchmarks are used. Hyperparameter sensitivity analysis (τΔ) is conducted on NQ separately, with the chosen value applied uniformly to all datasets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports per-benchmark scores across six benchmarks covering three task categories (knowledge-intensive QA, arithmetic reasoning, machine translation). Section 4.3 analyzes per-task word commitment distributions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses GSM8K underperformance (79.15 vs. LLaMA's 81.05), attributing it to base model disparity. Table 2 shows adding Qwen3-8B as a 4th model causes a performance drop. The Limitations section acknowledges open-source model restriction."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Adding a 4th model (Qwen3-8B) leads to a performance drop on NQ (Table 2). GSM8K performance below single-model LLaMA. Appendix F shows beam-search scaling degrades performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '6.88% average relative improvement' over strong ensemble baselines, which matches Table 1's improvement row. Claims about QA, arithmetic reasoning, and translation are all evaluated."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation studies (Sections 4.3-4.4) use controlled single-variable manipulation: replacing adaptive commitment with fixed-length alternatives while keeping other components constant, and toggling diversity-aware scaling. This supports the causal claims about component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'AdaFuse: Adaptive Ensemble Decoding with Test-Time Scaling for LLMs' suggests general LLM applicability. The abstract claims ADAFUSE 'consistently outperforms strong ensemble baselines' without qualification. Results are limited to 4 open-source ~8B parameter models on 6 benchmarks across 3 task types. While the Limitations section notes the open-source restriction, it does not bound claims to the tested model sizes, task types, or number of models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes GSM8K underperformance to 'large performance disparity between base models' but does not consider alternative explanations (e.g., tokenization differences, prompt sensitivity). No systematic discussion of confounds or alternative explanations for the observed improvements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: LLaMA-3.1-8B-Instruct, Mistral-7B-Instruct-v0.3, Qwen3-8B, InternLM3-8B-Instruct. These are specific enough to identify exact model checkpoints."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Only shot counts are mentioned: 5-shot for NQ/SQuAD/TriviaQA, 4-shot CoT for GSM8K, 0-shot for FLORES. No actual prompt text or few-shot examples are provided in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters reported: confidence threshold τΔ=0.7, max committed words M=3, branching factor B, context length limit up to 512 tokens. Sensitivity analysis for τΔ in Figure 6 and Appendix D.2."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "ADAFUSE is a decoding algorithm, not an agentic scaffolding system. No tools, retry logic, memory management, or agentic workflows are used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses standard benchmark splits but does not document preprocessing steps. Appendix C lists dataset sizes (e.g., 'SQuAD 2500') without explaining how the 2500 subset was selected from the full SQuAD dataset."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after the Conclusion, discussing the requirement for token-level likelihoods and restriction to open-source models."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section contains only one specific point: incompatibility with closed-source APIs. No discussion of threats specific to the evaluation methodology, such as benchmark contamination risk, model size generalization, sensitivity to model pairing, or representativeness of chosen benchmarks."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. The Limitations section notes the open-source requirement but does not specify that results may not generalize to larger models, other task types, other languages, or different model pairings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All benchmarks (NaturalQuestions, SQuAD, TriviaQA, GSM8K, FLORES) are publicly available for download. Independent researchers can verify results using these standard datasets."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks and cites their original papers. Appendix C provides descriptive statistics (number of entries, average lengths) for each dataset."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The paper uses pre-existing standard benchmark datasets. Recruitment methods do not apply."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document how benchmark subsets were selected. For example, 'SQuAD 2500' implies a subset of the full SQuAD dataset, but no selection criteria or filtering process is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure is present in the paper. The work used approximately 500 A100 GPU-hours but the funding source is not mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with the University of Illinois Urbana-Champaign, clearly stated on the title page. The paper does not evaluate any product made by the authors' institution."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding source is disclosed, so funder independence cannot be assessed. Since no funder is identified, this question is structurally unanswerable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses LLaMA-3.1-8B-Instruct, Mistral-7B-Instruct-v0.3, Qwen3-8B, and InternLM3-8B-Instruct to evaluate on standard benchmarks but does not state the training data cutoff dates for any model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. The benchmarks (NQ 2019, SQuAD 2016, TriviaQA 2017, GSM8K 2021) predate the training cutoffs of all tested models, making contamination a real risk that is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All QA and math benchmarks predate the likely training cutoffs of the tested models. No contamination analysis, canary strings, or temporal splits are discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 reports wall-clock inference time for ADAFUSE vs. baselines on 4x NVIDIA A100 80GB GPUs (Figure 5). Appendix A discusses GPU energy consumption tradeoffs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.1 states 'The total compute budget for all experiments is approximately 500 A100 GPU-hours.' Hardware (NVIDIA A100 80GB) and model memory footprints (~30GB in 16-bit) are specified."
       }
     }
   }
 }