ai-research-survey

scan.json (24950B)

---

 {
   "paper": {
     "title": "COLT: Lightweight Multi-LLM Collaboration through Shared MCTS Reasoning for Model Compilation",
     "authors": [
       "Annabelle Sujun Tang",
       "Christopher Priebe",
       "Lianhui Qin",
       "Hadi Esmaeilzadeh"
     ],
     "year": 2026,
     "venue": "Preprint",
     "arxiv_id": "2602.01935"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Code is available at https://github.com/he-actlab/COLT' on page 1 (footnote)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks are drawn from publicly available models (Llama-3-8B, DeepSeek-R1, FLUX, Llama-4-Scout) and the framework is built on Apache TVM, a public framework. The computational kernels are standard layers extracted from public model architectures."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Apache TVM v0.20.0 and an Intel Core i9 CPU and NVIDIA 2080 Ti GPU as hardware, but does not provide a requirements.txt, Dockerfile, or detailed dependency/version listing sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README with commands to run or a 'Reproducing Results' section is described in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports mean performance over 10 runs (Section 4.1: 'We repeat each experiment 10 times and report the mean performance') but does not report confidence intervals or error bars on any figures or tables."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims COLT outperforms baselines (e.g., '20.8% improvement') but no statistical significance tests are reported. Comparisons are made by directly comparing mean speedup numbers without p-values or any statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context. For example, 'COLT(8 Models) attains 14.98x final speedup, while GPT-5.2 only reaches 12.40x final speedup, a 20.8% improvement' (Section 4.2). Sample efficiency gains are also reported as ratios (e.g., 1.55x)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 5 benchmarks and 10 repetitions per experiment but provides no justification for why these numbers were chosen or any power analysis. No discussion of whether 5 benchmarks or 10 runs is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite repeating each experiment 10 times, the paper reports only mean performance. No standard deviation, error bars, or any spread measure is reported in any table or figure."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two single-model baselines are included: GPT-5.2 (single large) and gpt-5-mini (single small), using the same MCTS procedure with model selection disabled (Section 4.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines use GPT-5.2 (2025) and gpt-5-mini, which are contemporary models. The paper also compares against Llama-3.3-70B-Instruct as an alternative largest model. Prior LLM-guided compiler works (Pan et al., 2025; Tang et al., 2025) are cited as related."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablation studies: Section 4.3 ablates the choice of largest model (GPT-5.2 vs Llama-3.3-70B-Instruct), Section 4.4 ablates the hardware platform (CPU vs GPU), and the scaling from 2 to 4 to 8 models serves as an implicit ablation of model set size."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple metrics: execution latency/speedup, sample efficiency, sample efficiency gain, model invocation rate, and course alteration rate (Section 4.1)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for compiler optimization. The quality metric is execution latency, which is measured directly on hardware."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a learning task with train/test splits. The paper evaluates compiler optimization on specific computational kernels; there is no held-out test set concept applicable here."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (5 individual layers) across all figures and tables. Table 1 shows per-benchmark invocation rates. Tables 3-5 in the appendix provide per-layer, per-model call counts."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss specific failure cases or where the approach breaks down. No qualitative analysis of failed optimization trajectories or examples where COLT underperformed is provided."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every COLT configuration outperforms every single-model baseline across all benchmarks. No negative results, failed configurations, or approaches that were tried and abandoned are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims COLT achieves 10.86x speedup on CPU (30.05x on GPU) and outperforms the single-model baseline with only 23.9% calls to the largest LLM. These are supported by Figure 2, Figure 4, and Table 1 in the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about collaboration improving performance (e.g., 'collaboration enhances the quality of proposed transformations'). The ablation studies (varying model set size, largest model, hardware) provide controlled single-variable manipulations that support these claims, as all other search parameters are held constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests on 5 computational kernels from 4 model architectures on 2 hardware platforms, but the title and abstract claim broad applicability for 'model compilation' without bounding generalization. No discussion of whether results extend to other compiler frameworks, non-neural workloads, or other hardware targets beyond the two tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why collaboration helps. For example, it does not consider whether the improvement comes simply from increased diversity of proposals (more random seeds) rather than from the specific MCTS-based collaboration mechanism, or whether increased total compute budget explains the gains."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-5.2' and 'gpt-5-mini' without specifying API versions or snapshot dates. Llama-3.3-70B-Instruct and other open-source models are named but without specific checkpoint identifiers or version hashes. Marketing names alone do not count as specified versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides the full system prompt and a detailed example of the user prompt including code, transformation history, predicted scores, search context, per-model stats, local model context, and the expected JSON output format."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports key hyperparameters: lambda=0.5, exploration parameter c=sqrt(2), branching factor B=2. The LLM API parameters (temperature, top-p, etc.) are not reported, however the MCTS-specific hyperparameters are clearly stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The MCTS-based scaffolding is described in detail throughout Sections 2-3, including tree policy (Section 3.2), contextual model selection (Section 3.3), and course alteration mechanism (Section 3.4). Figure 1 provides an overview diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how computational kernels are extracted from the production models, how TVM IRModules are constructed, or what preprocessing steps are applied before optimization begins."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations section. The paper has an 'Impact Statement' section that is a single paragraph about making compilation more efficient, but no substantive discussion of limitations or threats to validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No discussion of which hardware targets, compiler frameworks, or workload types are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental data (per-run speedup values, optimization trajectories, timing measurements) are not available. Only aggregated means are reported in the paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the experimental setup: benchmarks are specific layers from named models, target hardware is specified (Intel Core i9, NVIDIA 2080 Ti), performance is measured as execution latency on target hardware, and each experiment is repeated 10 times."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard computational kernels from public model architectures."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from model architecture to computational kernel to TVM IRModule to optimization is not fully documented. How the initial programs are extracted and what preprocessing they undergo before entering the MCTS loop is not described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned anywhere in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with University of California, San Diego (footnote 1). The paper uses OpenAI APIs (GPT-5.2, gpt-5-mini) but the authors are not affiliated with OpenAI."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of funding disclosure is not the same as being unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate pre-trained model capability on a benchmark. It uses LLMs as proposal generators within an optimization loop; the LLMs are not being tested on memorized knowledge but on their ability to propose compiler transformations in context."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper does not evaluate LLMs on a benchmark where memorization would be a concern. The LLMs are generating transformation proposals conditioned on current program state, not recalling benchmark answers."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — contamination is not a relevant concern for this type of system where LLMs are used as optimization proposal generators rather than being evaluated on benchmark knowledge."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report API costs, tokens consumed, or wall-clock time for the optimization process despite calling GPT-5.2 and other models hundreds of times per experiment. Model invocation counts are reported (Tables 3-5) but not the associated dollar cost or latency."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report total API spend, total wall-clock time for the experiments, or GPU hours used for hardware-side evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "COLT(8 Models) achieves 10.86x average speedup on CPU and 30.05x on GPU across five benchmarks, outperforming the single-large-model baseline (GPT-5.2).",
       "evidence": "Figures 2 and 4 show speedup curves across five benchmarks on CPU and GPU. The abstract and Section 4.2 report these averaged numbers.",
       "supported": "moderate"
     },
     {
       "claim": "COLT reduces calls to the largest LLM to 23.9% of total calls (including course alteration) while outperforming using only the largest model.",
       "evidence": "Table 1 shows GPT-5.2 accounts for 12.3% of regular invocations and 23.9% total (including course alteration) for COLT(8 Models) on CPU.",
       "supported": "moderate"
     },
     {
       "claim": "Performance improves monotonically as the candidate model set is expanded from 2 to 4 to 8 models.",
       "evidence": "Figure 2 shows COLT(8 Models) >= COLT(4 Models) >= COLT(2 Models) in final speedup across all five CPU benchmarks.",
       "supported": "moderate"
     },
     {
       "claim": "COLT is robust to the choice of largest model, working with both proprietary GPT-5.2 and open-source Llama-3.3-70B-Instruct.",
       "evidence": "Section 4.3 and Figure 3 show COLT(8 Models) achieves the best final speedup on every benchmark with Llama-3.3-70B-Instruct as largest model, improving over the baseline by 4.8-27.5%.",
       "supported": "moderate"
     },
     {
       "claim": "COLT generalizes across hardware platforms (CPU and GPU).",
       "evidence": "Section 4.4 and Figure 4 show COLT consistently outperforms single-model baselines on NVIDIA 2080 Ti GPU.",
       "supported": "moderate"
     },
     {
       "claim": "Collaboration enhances the quality of proposed transformations compared to a single model, with large early-budget gains.",
       "evidence": "Section 4.2: 'on FLUX Convolution Layer at 100 samples, COLT(8 Models) reaches 3.83x speedup, a 49.6% improvement over the single-large-model baseline.' Similar early gains are shown in other benchmarks.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "COLT proposes a lightweight multi-LLM collaboration framework for compiler optimization that embeds model selection within a shared MCTS tree, enabling smaller models to drive most optimization decisions while selectively escalating to larger models. Across five computational kernels on CPU and GPU hardware, COLT consistently outperforms single-model baselines, achieving up to 10.86x average speedup on CPU with the largest model accounting for only 23.9% of total invocations. Performance scales monotonically with the number of collaborative models (2, 4, 8) and the approach is robust to substituting different largest models (GPT-5.2 vs Llama-3.3-70B-Instruct).",
   "red_flags": [
     {
       "flag": "No variance or uncertainty reporting despite 10 runs",
       "detail": "The paper explicitly states each experiment is repeated 10 times but reports only mean values. No standard deviations, error bars, or confidence intervals are provided in any figure or table, making it impossible to assess whether the reported improvements are statistically meaningful."
     },
     {
       "flag": "No negative results",
       "detail": "Every COLT configuration outperforms every baseline on every benchmark. No failed configurations, approaches that didn't work, or settings where collaboration hurt performance are reported, which is suspicious for a system with multiple hyperparameters and design choices."
     },
     {
       "flag": "No limitations discussion",
       "detail": "The paper lacks any limitations section, threats-to-validity discussion, or explicit scoping of what the results do not show. The Impact Statement is a single paragraph that does not address limitations."
     },
     {
       "flag": "Missing API cost analysis",
       "detail": "The paper's central claim is about lightweight collaboration, but no API costs or wall-clock times are reported. Without knowing the cost of hundreds of GPT-5.2 calls per experiment, it is impossible to assess practical viability or compare against the cost of simply running the single-large-model baseline longer."
     },
     {
       "flag": "Narrow benchmark coverage",
       "detail": "Results are from 5 computational kernels (attention, MoE, convolution, MLP layers) on only 2 hardware platforms, yet the claims are framed broadly for 'compiler optimization' and 'model compilation' without bounding generalization."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models for compiler optimization",
       "authors": ["C. Cummins", "V. Seeker", "D. Grubisic", "M. Elhoushi", "Y. Liang", "B. Roziere", "J. Gehring", "F. Gloeckle", "K. Hazelwood", "G. Synnaeve", "H. Leather"],
       "year": 2023,
       "relevance": "Foundational work on using LLMs for compiler optimization, establishing the paradigm that COLT extends to multi-model settings."
     },
     {
       "title": "LLM Compiler: Foundation language models for compiler optimization",
       "authors": ["C. Cummins", "V. Seeker", "D. Grubisic", "B. Roziere", "J. Gehring", "G. Synnaeve", "H. Leather"],
       "year": 2025,
       "relevance": "Trains foundation models specifically for compiler optimization, representing the single-large-model approach that COLT aims to outperform."
     },
     {
       "title": "CompilerDream: Learning a compiler world model for general code optimization",
       "authors": ["C. Deng", "J. Wu", "N. Feng", "J. Wang", "M. Long"],
       "year": 2025,
       "relevance": "Uses world models for compiler optimization, another approach to LLM-guided compilation that COLT builds upon."
     },
     {
       "title": "Compiler-R1: Towards agentic compiler auto-tuning with reinforcement learning",
       "authors": ["H. Pan", "H. Lin", "H. Luo", "Y. Liu", "K. Yao", "L. Zhang", "M. Xing", "Y. Wu"],
       "year": 2025,
       "relevance": "Applies reinforcement learning to agentic compiler optimization, directly relevant as a prior single-LLM approach for compilation."
     },
     {
       "title": "REASONING COMPILER: LLM-guided optimizations for efficient model serving",
       "authors": ["S. Tang", "C. Priebe", "R. Mahapatra", "L. Qin", "H. Esmaeilzadeh"],
       "year": 2025,
       "relevance": "Prior work by the same group on LLM-guided compiler optimization using MCTS, the direct predecessor to COLT."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2023,
       "relevance": "Pioneering work on cost-aware LLM usage through cascades and model selection, directly relevant to COLT's multi-model cost reduction goal."
     },
     {
       "title": "Hybrid LLM: Cost-efficient and quality-aware query routing",
       "authors": ["D. Ding", "A. Mallick", "C. Wang", "R. Sim", "S. Mukherjee", "V. Ruhle", "L. V. Lakshmanan", "A. H. Awadallah"],
       "year": 2024,
       "relevance": "Addresses cost-efficient model routing between different LLMs, a core challenge that COLT solves differently via endogenous MCTS-based selection."
     },
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Y. Du", "S. Li", "A. Torralba", "J. B. Tenenbaum", "I. Mordatch"],
       "year": 2024,
       "relevance": "Multi-LLM collaboration through debate, representing the explicit-communication approach that COLT's implicit shared-tree collaboration contrasts with."
     },
     {
       "title": "SLM-MUX: Orchestrating small language models for reasoning",
       "authors": ["C. Wang", "Z. Wan", "H. Kang", "E. Chen", "Z. Xie", "T. Krishna", "V. J. Reddi", "Y. Du"],
       "year": 2026,
       "relevance": "Contemporary work on orchestrating small LLMs for reasoning tasks, directly relevant to COLT's approach of using primarily small models."
     },
     {
       "title": "Mixture-of-agents enhances large language model capabilities",
       "authors": ["J. Wang", "J. Wang", "B. Athiwaratkun", "C. Zhang", "J. Zou"],
       "year": 2025,
       "relevance": "Multi-LLM collaboration via mixture-of-agents, relevant as an alternative collaboration paradigm to COLT's shared-tree approach."
     },
     {
       "title": "Agentic AI: a comprehensive survey of architectures, applications, and future directions",
       "authors": ["M. Abou Ali", "F. Dornaika", "J. Charafeddine"],
       "year": 2025,
       "relevance": "Survey of agentic AI architectures, relevant context for COLT's claim of being lightweight compared to conventional agentic machinery."
     },
     {
       "title": "A unified approach to routing and cascading for LLMs",
       "authors": ["J. Dekoninck", "M. Baader", "M. Vechev"],
       "year": 2025,
       "relevance": "Unified framework for LLM routing and cascading, directly relevant to COLT's model selection mechanism."
     }
   ]
 }