ai-research-survey

scan-v5.json (25915B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "ACAR: Adaptive Complexity Routing for Multi-Model Ensembles with Auditable Decision Traces",
     "authors": [
       "Ramchand Kumaresan"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.21231",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: σ routing achieves 55.6% (Table 1), exceeds Arena-2's 54.4% (Table 1), avoids full ensemble on 54.2% of tasks (32.9%+21.3% from Section 5.3), and retrieval decreases accuracy by 3.4pp (Table 2).",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal routing decisions are mechanistically specified in Algorithm 1: σ value deterministically selects execution mode. Retrieval harm is explained by median similarity of 0.167 (Figure 9), providing mechanistic justification for negative finding.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Results bounded to 4 benchmarks and 3 proprietary models (Claude, GPT-4o, Gemini). Abstract claims model-agnostic design but Section 8 limitations state may not generalize to open-source models and SuperGPQA dominates (66%).",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Paper discusses why σ-based approach chosen over learned routers (auditability trade-off, Section 3.2.3), why retrieval failed (weak similarity, Section 6.1), and mechanistic reasons for agreement-but-wrong failure (Section 6.2).",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Measures accuracy on ground-truth answers (pass rates, code execution verification) and cost in USD. Claims match measurement granularity. Acknowledges code equivalence issue (Section 8) but does not claim code equivalence as accuracy.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Dedicated Section 8 'Limitations' with four specific constraints, plus Section 6 'Negative Results and Failure Modes' documenting three systematic failures beyond scope.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 specifies: proprietary models only (no open-source), SuperGPQA dominance (66% of tasks), no learned router comparison, code equivalence inflation. Section 6.2 quantifies fundamental 8pp ceiling.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit boundaries: 1,510 tasks across 4 named benchmarks, 3 specific models, deterministic evaluation at temperature 0, σ routing with N=3 samples. What does NOT show: generalization to open-source models, to other model counts, to different problem domains.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section or acknowledgments disclosing financial support. Paper does not state whether funded by academic institution, company, or independent.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "Author affiliation not stated in paper header or acknowledgments. Single author 'Ramchand Kumaresan' with no institution listed.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "Cannot assess; funding not disclosed. Paper evaluates Claude (Anthropic), GPT-4o (OpenAI), Gemini (Google) — potential conflict if author affiliated with any provider, but affiliation unclear.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. Paper discloses AI assistance (Claude for code, ChatGPT for writing) but not financial interests, patents, or equity stakes.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Self-consistency variance σ defined formally (Definition 1, Eq. 1), execution mode M(σ) defined (Definition 2), TEAMLLM substrate described (Section 3.1), ACAR routing procedure in Algorithm 1.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions explicitly stated in Section 1.2: (1) ACAR routing mechanism with empirical results, (2) negative results on retrieval and attribution, (3) TEAMLLM reproducible infrastructure. Abstract and introduction clearly frame as 'measurement framework.'",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 engages with three research areas: multi-model routing (RouterBench, FrugalGPT, RouteLLM), cost-aware inference, reproducible benchmarking. Section 2.3 explicitly states how ACAR differs from learned routers, observability platforms, and benchmark papers.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Code and TEAMLLM substrate released at https://github.com/mechramc/ACAR-TeamLLM (stated in Section 3.1 and Appendix A). Figure regeneration scripts included.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "All 7,550+ runs (1,510 ACAR-U, 1,510 ACAR-UJ, plus baselines) released as runs.jsonl with decision traces (Appendix B). Input benchmarks are public: LiveCodeBench, SuperGPQA, etc.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Paper logs environment fingerprints with each run (Section 3.1) but does not specify requirements.txt, Python version, or dependencies in the paper text. Environment specs must be inferred from GitHub repo.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Paper states 'All figures regenerable from released artifacts' but provides no step-by-step instructions in the paper. Artifact manifest (Appendix B) lists directories but not how to execute them. GitHub repo likely has README, but paper itself lacks instructions.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Table 1 reports point estimates (55.6%, 54.4%, etc.) with no confidence intervals or error bars. Figures 2-7 show bar/line charts without error bands. No uncertainty quantification.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "Comparative claims ('ACAR-U exceeds Arena-2 by 1.2pp') lack p-values or statistical significance tests. No tests for difference in accuracy between configurations.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Effect sizes reported: ACAR-U +1.2pp vs Arena-2 (Table 1), retrieval -3.4pp (Table 2), cost differences in USD. Escalation rates given as percentages (32.9% single, etc.).",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "1,510 tasks evaluated but no justification for this n. Section 4.1 explains benchmark selection as 'cover diverse task types' but no power analysis or sample size calculation provided.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Execution is deterministic (Section 3.1: 're-execution with identical inputs produces identical outputs'). Single runs per configuration reported; no standard deviations, confidence intervals, or repeated evaluations.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Four configurations compared: Single-Model, Arena-2, ACAR-U, Arena-3 (Table 1, Section 4.3). Ablation comparing ACAR-U vs ACAR-UJ (Table 2).",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": false,
           "justification": "Baselines (single-model, two-model, three-model ensembles) are simple but not comparative to other routing methods. Related work cites RouterBench, FrugalGPT, RouteLLM but none are evaluated. Paper acknowledges this as intentional design choice for auditability over optimization.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "ACAR-U (without retrieval) vs ACAR-UJ (with retrieval augmentation) evaluated separately. Table 2 shows -3.4pp effect of adding Jungler retrieval component.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Accuracy (pass rate), cost (USD), escalation rate (% per mode), latency (ms), per-benchmark performance, per-mode breakdown all reported across Sections 5.1-5.4.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "N/A — evaluation on benchmarks (MathArena, Reasoning Gym, LiveCodeBench with code execution, SuperGPQA with multiple choice). No human subjects.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": false,
           "answer": false,
           "justification": "N/A — paper evaluates models on standard public benchmarks; it does not train models or hold out test data. Models are evaluated off-the-shelf.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results broken down by benchmark (Figure 3: MathArena, Reasoning Gym, LiveCodeBench, SuperGPQA), by execution mode (Figure 5: single/lite/full), by similarity/hit rate (Figures 8-9).",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 6 'Negative Results and Failure Modes' documents three systematic failures: retrieval decreases accuracy (6.1, Table 2), agreement-but-wrong is unrecoverable (6.2), attribution proxies weak (6.3).",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Negative results explicitly reported: ACAR-UJ -3.4pp vs ACAR-U (Table 2), 8pp ceiling from agreement-but-wrong (Section 6.2), attribution proxies 'showed weak correlation' (Section 6.3). Conclusion states 'What failed' alongside 'What worked.'",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Models named: 'Claude Sonnet 4', 'GPT-4o', 'Gemini 2.0 Flash' (Section 4.2). No version hashes, snapshot dates, or training cutoff dates provided. Identifiable by name at publication time (2026-02) but not fully reproducible.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "Algorithm 1 mentions 'Mprobe(T)' sampling and EXTRACT function, but actual prompts/system instructions given to models are not shown. Section 3.1 mentions 'prompt template hash' is logged but template not disclosed.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "σ thresholds (0.0, 0.5, 1.0) specified in Definition 1, N=3 samples justified in Section 3.2.3, temperature=0 for all models (Section 4.2), retrieval threshold=0.0 for ACAR-UJ (Section 3.2.4), with discussion of why thresholds >0.7 needed (Section 6.1).",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "N/A — paper evaluates models directly on benchmarks. No agentic scaffolding (tool use, chain-of-thought, step-by-step prompting) described. EXTRACT function handles answer canonicalization but not model scaffolding.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": false,
           "justification": "Algorithm 1 mentions EXTRACT(ri) for canonicalization but does not detail how answers are extracted/compared. Section 8 acknowledges 'LiveCodeBench escalation is inflated by syntactically different but semantically equivalent outputs' but does not document preprocessing steps.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "All runs (outputs) released as runs.jsonl with per-task decision traces. Appendix B lists: phase22_acar_u/runs.jsonl (1,510 ACAR-U runs), phase22_acar_uj/runs.jsonl, baseline runs (arena_3model, arena_2model, single_model).",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": false,
           "answer": false,
           "justification": "N/A — paper uses existing public benchmarks (MathArena, Reasoning Gym, LiveCodeBench, SuperGPQA). Does not describe collection of these benchmarks; evaluation section (4.1) describes benchmarks, not collection methodology.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "N/A — no human participants. Benchmarks are task sets, not recruited subjects.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.1 describes TEAMLLM execution substrate: deterministic execution with seed/hash logging, immutable append-only artifacts (runs.jsonl), forward-only state machine. Algorithm 1 documents full routing procedure from task T to decision trace D.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Models used (Claude Sonnet 4, GPT-4o, Gemini 2.0 Flash) are proprietary with unknown training data cutoffs. Paper does not state training dates or potential overlap with benchmark data.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Paper does not discuss whether benchmarks (MathArena, Reasoning Gym, LiveCodeBench, SuperGPQA) were in model training data. Potential contamination of proprietary models not addressed.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "LiveCodeBench mentioned as having 'temporal splits' (Section 2.3) but paper does not verify that tested models were trained before these benchmarks were created or evaluate contamination impact.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 reports cost in USD: Single-Model $17.04, Arena-2 $20.64, ACAR-U $20.34, Arena-3 $20.64. Cost vs. accuracy Pareto frontier shown in Figure 4.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Total compute budget stated: ACAR-U costs $20.34 total across 1,510 tasks (≈$0.013 per task). Section 5.2 compares cost-accuracy trade-off. Figure 6 shows cumulative cost progression.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "σ-based routing achieves 55.6% accuracy on 1,510 benchmark tasks",
       "evidence": "Table 1 reports ACAR-U accuracy of 55.6% (839/1510 correct)",
       "supported": "strong"
     },
     {
       "claim": "σ-based routing exceeds two-model baseline (Arena-2) by 1.2 percentage points while costing 1.5% less",
       "evidence": "Table 1: ACAR-U 55.6% ($20.34) vs Arena-2 54.4% ($20.64)",
       "supported": "strong"
     },
     {
       "claim": "ACAR avoids full ensembling on 54.2% of tasks by routing to single-agent or two-model modes",
       "evidence": "Section 5.3 reports escalation: 32.9% single-agent + 21.3% arena-lite = 54.2%, 45.8% full-arena",
       "supported": "strong"
     },
     {
       "claim": "Retrieval augmentation with low-quality stores decreases accuracy by 3.4 percentage points",
       "evidence": "Table 2 shows ACAR-UJ 52.4% vs ACAR-U 55.6%, difference -3.4pp across all benchmarks",
       "supported": "strong"
     },
     {
       "claim": "When models unanimously agree on incorrect answers (σ=0), no downstream ensemble can recover; this bounds achievable accuracy at 8pp below full ensembling",
       "evidence": "Section 6.2 explains 'agreement-but-wrong' as intrinsic to self-consistency; ACAR-U 55.6% vs Arena-3 63.6% = 8pp gap",
       "supported": "strong"
     },
     {
       "claim": "Attribution proxies (response similarity, entropy) showed weak correlation with ground-truth leave-one-out values",
       "evidence": "Section 6.3 states proxies 'showed weak correlation with ground-truth leave-one-out values; practical attribution requires explicit counterfactual computation'",
       "supported": "moderate"
     },
     {
       "claim": "σ-routing mechanism is model-agnostic and requires no learned components",
       "evidence": "Algorithm 1 shows deterministic routing based on σ; Section 3.2.3 justifies choice to avoid distribution shift. Only tested on 3 proprietary models.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "ACAR proposes σ-based adaptive routing using self-consistency variance to allocate compute across multi-model ensembles. On 1,510 benchmark tasks, σ-routing achieves 55.6% accuracy—exceeding two-model ensembling by 1.2pp while avoiding full ensemble on 54% of tasks. The paper documents three critical failures: naive retrieval augmentation hurts (-3.4pp) without task-aligned semantic thresholds (>0.7); the algorithm has an irreducible 8pp ceiling when all models agree incorrectly; and post-hoc attribution from proxy signals does not correlate with ground truth.",
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "Evaluates proprietary models (Claude, GPT-4o, Gemini) with unknown training cutoffs on benchmarks; does not discuss potential overlap between model training data and evaluation benchmarks."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Reports point estimates (55.6% vs 54.4%) without confidence intervals, p-values, or error bars. ACAR-U leads by only 1.2pp; statistical significance unclear given no multiple trials."
     },
     {
       "flag": "Deterministic runs without uncertainty quantification",
       "detail": "Single deterministic execution per configuration (by design). No repeated evaluations or confidence intervals. Uncertainty in model outputs not measured."
     },
     {
       "flag": "Prompts not disclosed",
       "detail": "Algorithm and hyperparameters described, but actual prompts/system instructions given to models are not provided. Prompt hashes logged but templates hidden, limiting reproducibility."
     },
     {
       "flag": "Limited baseline comparisons",
       "detail": "No comparison to other routing methods (RouterBench, FrugalGPT, RouteLLM) mentioned in related work. Only compared to naive ensemble baselines."
     },
     {
       "flag": "Weak evidence for attribution failure",
       "detail": "Section 6.3 claims attribution proxies 'showed weak correlation' but provides no figures, correlation coefficients, or detailed analysis. Minimal supporting evidence."
     },
     {
       "flag": "Benchmark dominance not controlled",
       "detail": "SuperGPQA comprises 66% of tasks (1,000/1,510); results heavily skewed toward knowledge-based multiple-choice. No stratified or weighted analysis."
     },
     {
       "flag": "Missing funding/affiliation disclosure",
       "detail": "No funding source stated. Single author with no institutional affiliation listed. Paper evaluates three major LLM providers; conflict-of-interest status unclear."
     }
   ],
   "cited_papers": [
     {
       "title": "RouterBench: A benchmark for multi-LLM routing system",
       "relevance": "Benchmark for evaluating LLM routing systems; ACAR differs by using heuristic σ-based routing instead of learned classifiers"
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "relevance": "Cascading cost-aware routing strategy; ACAR compares to this approach for cost-quality trade-offs"
     },
     {
       "title": "RouteLLM: Learning to route LLMs with preference data",
       "relevance": "Preference-learning based routing; ACAR explicitly avoids learned routers for interpretability"
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "relevance": "Single-model agentic reasoning with tools; related to multi-model orchestration but focuses on tool use within one model"
     },
     {
       "title": "A survey on mixture of experts in large language models",
       "relevance": "Token-level routing within a single model; orthogonal to inter-model orchestration"
     },
     {
       "title": "LiveCodeBench: A challenging benchmark for code generation with execution-based verification",
       "relevance": "Execution-verified code evaluation benchmark used to evaluate ACAR routing on deterministic code tasks"
     },
     {
       "title": "The Shapley value in machine learning",
       "relevance": "Attribution method for assigning credit in multi-agent systems; ACAR discusses why Shapley-like proxies fail"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Cost-quality trade-offs are immediately relevant to practitioners deploying multi-model LLM systems, but requires access to three different proprietary APIs simultaneously."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges assumptions: retrieval augmentation with weak semantic alignment hurts (not helps), attribution from proxy signals doesn't work, and unanimous model agreement is an irreducible failure mode."
     },
     "fear_safety": {
       "score": 0,
       "justification": "Paper focuses on cost-efficiency and routing optimization. No AI safety, alignment, or risk concerns raised or addressed."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Technical contribution lacks narrative drama. 'Agreement-but-wrong' failure is intellectually interesting but not emotionally engaging."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and artifacts released on GitHub; results are reproducible from provided runs.jsonl. However, requires API keys for three proprietary models, limiting who can reproduce live."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Single author with no stated affiliation. TEAMLLM is novel but not established brand. Paper likely to appeal to infra-focused practitioners rather than general audience."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "47154950",
         "title": "Aletheia Tackles FirstProof Autonomously",
         "points": 5,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=47154950",
         "created_at": "2026-02-25T17:46:36Z"
       },
       {
         "hn_id": "47314080",
         "title": "Latent Context Compilation: Distilling Long Context into Compact Portable Memory",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=47314080",
         "created_at": "2026-03-09T19:21:30Z"
       }
     ],
     "top_points": 5,
     "total_points": 7,
     "total_comments": 0
   }
 }