ai-research-survey

calibration.json (18324B)

---

 {
   "paper_slug": "acar-adaptive-complexity-2026",
   "total_questions": 50,
   "agreement_count": 48,
   "disagreement_count": 2,
   "agreement_rate": 0.96,
   "disagreements": [
     {
       "category": "statistical_methodology",
       "question": "effect_sizes_reported",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": true,
       "opus_justification": "The paper reports percentage-point differences with full baseline context (e.g., ACAR-U 55.6% vs Arena-2 54.4%, retrieval degradation from 55.6% to 52.4% = -3.4pp). The schema explicitly states: 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' Table 1 provides all absolute numbers alongside the differences, meeting this threshold.",
       "sonnet_justification": "Effect sizes are reported only as raw percentage-point differences (e.g., '-3.4pp' for retrieval augmentation), but without baseline context or standardized effect size measures (Cohen's d, etc.) and without any statistical tests making these differences interpretable.",
       "direction": "opus_generous"
     },
     {
       "category": "data_integrity",
       "question": "data_pipeline_documented",
       "sonnet_applies": true,
       "sonnet_answer": true,
       "opus_applies": true,
       "opus_answer": false,
       "opus_justification": "The main execution pipeline (task -> ACAR routing -> results) is documented via Algorithm 1 and the TEAMLLM substrate. However, the experience store for ACAR-UJ (837 entries) appears with no explanation of how it was constructed. The benchmark task selection process (whether 60 MathArena tasks, 250 Reasoning Gym tasks, etc. are the full benchmarks or subsamples) is also not explicitly stated. These are unexplained elements in the data pipeline.",
       "sonnet_justification": "The full pipeline is documented: Algorithm 1 specifies the routing procedure, Section 3.1 describes TEAMLLM's data capture (seed, prompt hash, environment fingerprint per run), and Appendix A describes how figures can be regenerated from the runs.jsonl files. The pipeline from task to final answer is fully specified.",
       "direction": "sonnet_generous"
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub URL (https://github.com/mechramc/ACAR-TeamLLM) in a footnote in Section 1.2 and again in Appendix A, described as publicly available with all code and figure regeneration scripts."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses four publicly available benchmarks (MathArena, Reasoning Gym, LiveCodeBench, SuperGPQA) and additionally releases execution artifacts (runs.jsonl with 7,550+ runs) via the GitHub repository as described in Appendix B."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'environment fingerprint' as part of run logging (Section 3.1, Appendix A) but does not provide a requirements.txt, Dockerfile, conda environment file, or library version listing in the paper. No dependency specifications are given."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A states 'All figures in this paper can be regenerated from released artifacts using provided scripts.' Appendix B provides a detailed artifact manifest showing the file structure. The GitHub repository is referenced as containing all regeneration scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports only point estimates (e.g., '55.6% accuracy', '54.4%') with no confidence intervals, error bars, or uncertainty notation in any table or figure."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'ACAR-U exceeds Arena-2 by 1.2 percentage points' without any statistical significance test. No p-values, t-tests, bootstrap tests, or other significance measures appear anywhere in the paper."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage-point differences with full baseline context: ACAR-U 55.6% vs Arena-2 54.4% (1.2pp improvement), ACAR-UJ 52.4% vs ACAR-U 55.6% (-3.4pp). Table 1 provides absolute numbers for all configurations. Per the schema, 'percentage improvement with baseline context' satisfies this criterion."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 1,510 tasks across four benchmarks with no justification for why this sample size is sufficient. No power analysis or discussion of statistical adequacy is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses temperature=0 for determinism but reports single-run results with no variance, standard deviation, or spread measures. No analysis of variance from any source (model non-determinism, benchmark sampling) is provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared against in Table 1: Single-Model (best single model, Claude Sonnet 4), Arena-2 (two-model ensemble), and Arena-3 (three-model ensemble on all tasks)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines use current frontier models (Claude Sonnet 4, GPT-4o, Gemini 2.0 Flash) from major providers as of 2026. The multi-model ensemble baselines represent current practice."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "ACAR-U vs ACAR-UJ ablates the Jungler retrieval component. Table 2 provides per-benchmark results for both configurations, isolating the retrieval component's effect."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports accuracy (Table 1), total API cost in USD (Table 1, Figure 4, Figure 6), escalation rate by mode (Section 5.3, Figure 5), and latency in milliseconds (Figure 7)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to this paper's claims. All benchmarks use automated correctness verification (test case execution for LiveCodeBench, answer matching for others)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "All four benchmarks are used as evaluation sets. ACAR has no learned components and performs no tuning on these benchmarks, so all reported results are effectively on held-out data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark in Figure 3 (pass rates), Table 2 (retrieval impact), and Figure 5 (escalation distribution). Individual benchmark performance varies substantially (SuperGPQA 60.5% vs MathArena 26.7%)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is dedicated to negative results and failure modes: retrieval augmentation hurting performance (6.1), agreement-but-wrong being unrecoverable (6.2), and attribution proxies failing (6.3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports three quantified negative findings: retrieval augmentation decreases accuracy by 3.4pp (Table 2), agreement-but-wrong creates an 8pp ceiling (Section 6.2), and attribution proxies show weak correlation (Section 6.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: 55.6% accuracy (Table 1), 54.2% avoidance of full ensembling (Section 5.3), -3.4pp retrieval degradation (Table 2), 8pp gap to Arena-3 (Table 1), attribution proxy failure (Section 6.3)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that retrieval hurts performance is supported by controlled ablation: ACAR-U and ACAR-UJ differ only in the retrieval component, tested on the same 1,510 tasks. This single-variable manipulation is adequate for the causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly bounds generalization: 'Three models from major providers; may not generalize to open-source models' and 'Benchmark bias: SuperGPQA dominates (66% of tasks).' The paper does not overclaim beyond its tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations for key findings: low semantic similarity (median 0.167) as the cause of retrieval failure rather than retrieval being inherently harmful (Section 6.1), syntactic code equivalence inflating LiveCodeBench escalation (Section 8), and the structural explanation for the 8pp accuracy ceiling (Section 6.2)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper names 'Claude Sonnet 4', 'GPT-4o', and 'Gemini 2.0 Flash' without API version identifiers or snapshot dates. Per the schema, marketing names without snapshot dates or API versions do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Algorithm 1 describes the routing procedure and the paper mentions 'prompt template hash' in run logging, but the actual prompt text sent to models is not provided in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature=0 is stated (Section 4.2), N=3 probe samples specified (Section 3.2.1), and the retrieval similarity threshold of 0.0 for ACAR-UJ is stated (Section 3.2.4)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "TEAMLLM substrate is described in detail (Section 3.1) with three enforced invariants. Algorithm 1 fully specifies the ACAR routing procedure including probing, routing, and logging phases. The Jungler retrieval component is described in Section 3.2.4."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states benchmark task counts (60+250+200+1000=1510) but does not document whether these are full benchmarks or subsamples, how tasks were selected from each benchmark, or how the experience store (837 entries) for ACAR-UJ was constructed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Limitations' is a dedicated section listing four specific limitations: model set coverage, benchmark bias, absence of learned routing comparison, and code equivalence issues."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 identifies specific threats: 'SuperGPQA dominates (66% of tasks)' affecting aggregate result generalizability, and 'LiveCodeBench escalation is inflated by syntactically different but semantically equivalent outputs' affecting routing efficiency measurements."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: results may not generalize to open-source models, learned routers may outperform on specific distributions, and the 'What holds / What does not hold' abstract structure clearly demarcates what is and isn't claimed."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Complete runs.jsonl files (7,550+ runs with per-task decision traces) are released via the GitHub repository. Appendix B provides the artifact manifest. Independent verification of all reported results is possible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the four benchmarks with task counts and characteristics. Section 3.1 describes TEAMLLM's data capture mechanism (seed, prompt hash, environment fingerprint per run). The execution procedure is documented in Algorithm 1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved. The paper uses standard public benchmarks for automated evaluation. Recruitment methods are not applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The execution pipeline (task -> ACAR routing -> results) is documented via Algorithm 1 and TEAMLLM. However, the experience store for ACAR-UJ (837 entries) appears with no construction explanation, and whether the benchmark task counts represent full benchmarks or subsamples is not stated. These are undocumented elements in the data pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned anywhere in the paper. There is no acknowledgments section listing grants, corporate sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists only the author name 'Ramchand Kumaresan' with no institutional affiliation. The author evaluates products from Anthropic, OpenAI, and Google without disclosing any relationship to these companies."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding source is disclosed. The paper appears to be solo independent research with no identified funder, making funder independence not applicable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement, patent disclosure, or financial interests declaration anywhere in the paper. Per the schema, absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the three models (Claude Sonnet 4, GPT-4o, Gemini 2.0 Flash). This information is needed to assess whether benchmark tasks could have appeared in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap for any of the four benchmarks. The paper does not analyze whether benchmark tasks appeared in model training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses four benchmarks without discussing whether they were available before the training cutoffs of the models tested. No contamination analysis or discussion is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study with automated metrics."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Demographics are not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Inclusion/exclusion criteria are not applicable."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Randomization of participant assignment is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports total API costs in USD for each configuration (Single-Model: $17.04, Arena-2: $20.64, ACAR-U: $20.34, Arena-3: $20.64). Figure 4 shows the cost-accuracy Pareto frontier. Per-run cost is logged in the decision trace."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Total API costs for all configurations are reported in Table 1. Figure 6 shows cumulative cost over tasks ($20.34 for ACAR-U). Latency distributions are also reported in Figure 7."
       }
     }
   }
 }