ai-research-survey

calibration.json (22649B)

---

 {
   "paper_slug": "adaptevolve-improving-efficiency-2026",
   "total_questions": 50,
   "agreement_count": 45,
   "disagreement_count": 5,
   "agreement_rate": 0.90,
   "disagreements": [
     {
       "category": "artifacts",
       "question": "environment_specified",
       "sonnet_applies": true,
       "sonnet_answer": true,
       "opus_applies": true,
       "opus_answer": false,
       "opus_justification": "The schema requires 'requirements.txt, Dockerfile, conda environment file, or a detailed Environment Setup section listing library versions.' The paper lists hyperparameters in Table 5 and mentions hardware (8x AMD MI250 GPUs), but does not provide a requirements.txt, Dockerfile, or library version listing. Citing scikit-learn and River without version numbers is insufficient. Hyperparameters are not the same as environment/dependency specifications.",
       "sonnet_justification": "Table 5 (Appendix A.4) specifies exact hyperparameters including model names (Qwen3-32B, Qwen3-4B), temperature (0.6), top-p (0.95), max tokens (20000), and hardware (8x AMD Instinct MI250 GPUs). The River package (Montiel et al., 2021) and scikit-learn (Buitinck et al., 2013) are cited. This provides sufficient detail for environment reconstruction.",
       "direction": "sonnet_generous"
     },
     {
       "category": "statistical_methodology",
       "question": "effect_sizes_reported",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": true,
       "opus_justification": "The schema says 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' The paper reports exactly this pattern: 'retains 97.9% of the 32B upper-bound accuracy (73.6% vs 75.2%) while reducing compute cost by 34.4%', '2.4 points (73.6% vs 71.2%)', and '41.5% while maintaining 97.1% of peak accuracy.' These provide percentage improvements with explicit baseline context, which satisfies the schema criterion.",
       "sonnet_justification": "The paper reports raw differences (e.g., '2.4 points', '34.4% cost reduction') but no standardized effect sizes such as Cohen's d or odds ratios.",
       "direction": "opus_generous"
     },
     {
       "category": "claims_and_evidence",
       "question": "causal_claims_justified",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": true,
       "opus_justification": "The schema states 'ablation studies (removing component X reduces performance by Y%) ARE causal claims — check whether the ablation design is adequate (controlled single-variable manipulation counts as YES).' The paper does controlled single-variable comparisons: static tree vs. adaptive tree (Table 1), threshold vs. tree (Table 4), and various routing strategies vs. pure baselines. Each comparison changes one variable while holding the benchmark, models, and other settings constant. This is adequate controlled experimental design for the causal claims made. The lack of statistical power (single run) is captured by other checklist items (significance_tests, variance_reported).",
       "sonnet_justification": "The paper claims confidence-driven routing 'reduces total inference compute by 37.9% while retaining 97.5% accuracy' and that 'intrinsic confidence provides a robust signal for safe agentic acceleration.' These are causal claims about what the method achieves, but without multiple seeds or statistical tests, alternative explanations (lucky single run, benchmark-specific effects) are not ruled out.",
       "direction": "opus_generous"
     },
     {
       "category": "conflicts_of_interest",
       "question": "funding_disclosed",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": true,
       "opus_justification": "The footnote on the first page states 'Work done during Internship at Advanced Micro Devices, Inc. (AMD).' This explicitly identifies AMD as the institution that supported the work. While there is no formal 'Acknowledgments' section with grant numbers, the funding source (AMD via internship) is clearly disclosed to the reader. The schema asks 'Is the funding source disclosed?' and it is.",
       "sonnet_justification": "The paper acknowledges '*Work done during Internship at Advanced Micro Devices, Inc. (AMD)' in a footnote, but there is no formal funding disclosure, acknowledgments section, or grant information provided.",
       "direction": "opus_generous"
     },
     {
       "category": "cost_and_practicality",
       "question": "compute_budget_stated",
       "sonnet_applies": true,
       "sonnet_answer": true,
       "opus_applies": true,
       "opus_answer": false,
       "opus_justification": "The schema asks 'Is the total computational budget stated? Look for: GPU hours, total API spend, hardware used, training time.' While Section 3.1 states the hardware (8x AMD MI250 GPUs) and Table 5 lists configuration parameters, the paper never states the TOTAL compute consumed — no GPU-hours, no wall-clock time for experiments, no total training/inference time. Listing hardware is necessary but not sufficient; the total budget (how long the hardware was used) is not quantified.",
       "sonnet_justification": "Section 3.1 states experiments were executed on 'a single node with 8x AMD Instinct MI250 GPUs.' Table 5 reports max iterations (8), evaluation timeout (1200s per evaluation), and parallel evaluations (32), enabling estimation of the compute budget.",
       "direction": "sonnet_generous"
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract provides a GitHub URL: 'Our code is available at https://github.com/raypretam/adaptive_llm_selection'. This is a working repository link, satisfying the criterion."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two publicly available standard benchmarks: LiveCodeBench v5 (Jain et al., 2024) with 880 samples and MBPP (Austin et al., 2021) with 974 samples. Neither benchmark was modified by the authors."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists hyperparameters in Table 5 and mentions hardware (8x AMD MI250 GPUs), but does not provide a requirements.txt, Dockerfile, conda environment file, or a detailed Environment Setup section with library versions. Citing scikit-learn and River without version numbers is insufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While code is released and hyperparameters are listed in Table 5, there is no 'Reproducing Results' section or README-like instructions within the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1, 2, and 4 are point estimates only. No confidence intervals, error bars, or +/- notation appear anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AdaptEvolve 'strictly outperforms' and 'significantly surpasses' baselines but provides no p-values, t-tests, or any statistical significance tests to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Per the schema, 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' The paper reports: 'retains 97.9% of the 32B upper-bound accuracy (73.6% vs 75.2%) while reducing compute cost by 34.4%' and '41.5% cost reduction while maintaining 97.1% of peak accuracy (91.3/94.0).' These provide percentage improvements with explicit baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The warm-up set of N=50 is described as 'minimal' but no justification or power analysis explains why 50 samples is sufficient to bootstrap a reliable routing classifier. No justification for benchmark sample sizes either."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 reports standard deviations for confidence metrics in calibration analysis, but the main experimental results (Tables 1, 2) are single-run point estimates with no variance across seeds or runs. The schema specifies: 'If the paper reports single-run numbers only, NO.'"
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against multiple baselines: Pure 4B (lower bound), Pure 32B (upper bound), Random Routing, Static Decision Tree, and the Cascading baseline from Chen et al. (2023). See Tables 1 and 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The Cascading baseline (Chen et al., 2023) is recent and represents a standard approach. Pure model bounds and random routing are appropriate reference points. While RouteLLM (2024) is cited but not compared, the included baselines are reasonable for the evolutionary agent setting."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares Static Decision Tree vs. Adaptive Hoeffding Tree on both benchmarks, isolating the contribution of online adaptation. Appendix A.3 (Table 4) compares the Decision Tree router against a threshold-based switch, testing the non-linear routing hypothesis."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports three metrics across all configurations: Accuracy, Compute Cost (normalized units), and Efficiency (Accuracy/Cost), plus the Small:Large usage ratio. Tables 1 and 2 present all metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates code generation using automated pass/fail test cases on benchmarks. Human evaluation is clearly irrelevant to claims about routing efficiency in an automated evolutionary system."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The main evaluation uses the full LiveCodeBench v5 (880 samples) and MBPP (974 samples) benchmarks, which are standard held-out test sets not seen during model training. The N=50 warm-up phase is part of the method's operation rather than hyperparameter tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported only as single aggregate numbers per benchmark (Table 1). No per-difficulty, per-category, or per-task breakdown is provided for either LiveCodeBench or MBPP."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or error analysis is presented. The paper does not discuss instances where the router made incorrect routing decisions, nor does it provide qualitative examples of failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment presented shows AdaptEvolve performing favorably. No configurations where the method performed worse than baselines or failed to improve are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '37.9% cost reduction' and '97.5% accuracy retention.' Table 1 and Section 3.6 support these: LiveCodeBench shows 34.4% cost reduction at 97.9% accuracy retention; MBPP shows 41.5% at 97.1%. The averages match the abstract claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation comparisons. Per the schema, 'controlled single-variable manipulation counts as YES.' The paper compares routing strategies while holding benchmarks, models, and other settings constant (e.g., static tree vs. adaptive tree in Table 1; threshold vs. tree in Table 4). This controlled experimental design is adequate for the specific causal claims about routing component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion claims 'intelligent resource allocation is a viable pathway for scalable agentic reasoning' — a broad generalization from two Qwen3 models on two Python coding benchmarks within one framework (OpenEvolve). The paper does not bound claims to the specific tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the observed results are discussed. The paper does not consider whether efficiency gains could stem from benchmark-specific properties, the Qwen3 model family characteristics, warm-up set selection effects, or other confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 specifies 'Qwen3-4B' and 'Qwen3-32B' from the Qwen3 family with citation to the technical report (Yang et al., 2025). For open-weight models, the model name plus parameter count identifies specific model releases."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.1 mentions that 'previous high-scoring generations serve as few-shot exemplars' but no actual prompt text sent to the models is provided anywhere in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 (Appendix A.4) provides a comprehensive hyperparameters table covering LLM settings (temperature=0.6, top-p=0.95, max tokens=20000), evolutionary parameters (population size=8, archive size=3), and confidence calculation windows (2048)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The adaptive routing scaffold is described in detail: Section 2 explains the decision tree router, confidence metric computation, and routing logic. Figure 1 provides a workflow diagram. Table 6 shows the labeling logic. Appendix A.1 details confidence computation formulas."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data preprocessing steps are documented. Section 3.3 states sample counts (880 for LiveCodeBench, 974 for MBPP) but provides no details on any filtering, preprocessing, or transformations applied to benchmark inputs before use in the evolutionary framework."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is titled 'Limitations' and substantively discusses that the method requires access to token-level log-probabilities, restricting applicability to open-weight models or APIs that expose logprobs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section discusses only a deployment constraint (logprob access requirement), which is a scope limitation rather than a threat to the validity of the reported results. No specific threats to validity — such as single-run result reliability, benchmark contamination risk, generalizability to other model families, or warm-up sample selection effects — are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There are no statements bounding the results to only Qwen3 models, only Python coding tasks, only OpenEvolve-style frameworks, or only the two tested benchmarks. The logprob limitation is about method applicability, not about what the results demonstrate."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregated results appear in tables. No raw per-sample predictions, routing decisions, confidence scores, or intermediate evolutionary outputs are released or available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes the benchmarks used (LiveCodeBench v5 with 880 samples, MBPP with 974 samples). Section 3.5 describes the warm-up data collection (N=50) with the labeling logic detailed in Table 6."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Data comes from standard public benchmarks (LiveCodeBench, MBPP), so participant recruitment does not apply."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The confidence computation pipeline is described (Appendix A.1), but the full data pipeline from benchmark input to final evaluation output is incompletely documented. Details on how evolutionary iterations proceed, how outputs are aggregated, and how final accuracy numbers are computed from the evolutionary population are not fully specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The footnote on the first page states 'Work done during Internship at Advanced Micro Devices, Inc. (AMD)', clearly identifying AMD as the institution that supported and funded the work."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the title page: Pretam Ray (IIT Kharagpur) and Pratik Prabhanjan Brahma, Zicheng Liu, Emad Barsoum (AMD). The footnote explicitly states the work was performed during an AMD internship."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "AMD funded the work via internship, three of four authors are AMD employees, and experiments were conducted on AMD MI250 GPUs. AMD has commercial interest in demonstrating efficient inference on its hardware. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears anywhere in the paper. AMD affiliations are disclosed but no formal declaration of conflicts of interest is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Qwen3-4B and Qwen3-32B on code benchmarks but does not state the training data cutoff date for either model. The Qwen3 technical report is cited but the cutoff date is not mentioned."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. MBPP (2021) is particularly at risk of being in Qwen3 training data given the multi-year gap, but this is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MBPP (Austin et al., 2021) predates Qwen3 by several years and is widely used in LLM training. LiveCodeBench is designed to be contamination-free but this property is not discussed in the paper. Neither benchmark's contamination risk is addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 and Tables 1-2 report inference cost in normalized compute units (one 32B call = 1.0, one 4B call = 0.125), with total costs per configuration reported across both benchmarks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.1 states the hardware (8x AMD MI250 GPUs) and Table 5 lists configuration parameters, but the total compute consumed is never stated — no GPU-hours, wall-clock time, or total experiment duration. Listing available hardware is not the same as quantifying the total computational budget used."
       }
     }
   }
 }