ai-research-survey

scan.json (24106B)

---

 {
   "paper": {
     "title": "CoLM: Collaborative Large Models via A Client-Server Paradigm",
     "authors": ["Siqi Huang", "Sida Huang", "Hongyuan Zhang"],
     "year": 2025,
     "venue": "AAAI 2026",
     "arxiv_id": "2511.06991"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper. No supplementary materials with code are referenced."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: MME, SEED Bench, MMBench, AI2D, OCRBench, MMMU, AlpacaEval 2.0, Arena-Hard, and MT-Bench. All are standard public benchmarks that were not modified."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is provided. The paper mentions models are accessed via APIs or run locally using 'open-source inference frameworks' but does not specify which frameworks, library versions, or hardware."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The method description gives a high-level overview of the pipeline but not enough detail to reproduce experiments without guessing."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-4 are reported as single point estimates with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., CoLM improves over baselines) based solely on comparing raw numbers. No statistical significance tests (t-tests, bootstrap tests, etc.) are used."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute improvements (delta values) in Table 3 and percentage/point improvements throughout (e.g., Janus-Pro-7B going from 47.55 to 58.06 average score). The baseline context is always provided alongside the improved results."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the choice of benchmarks, number of benchmarks, or why particular model combinations were selected. No power analysis or sample size discussion."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs. No standard deviation, variance across seeds, or multi-run statistics are reported anywhere in the paper."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against standalone model performance (each model without collaboration) and against MoA (Mixture-of-Agents, Wang et al. 2024a) in Table 2. The server output is also included as an upper bound reference."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "MoA (Wang et al. 2024a) is a recent and representative collaborative framework. The standalone models used (GPT-4o, Qwen2.5-VL-7B, Janus-Pro-7B) are contemporary models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes three ablation studies: (1) individual client contributions (Table 3), (2) effect of collaborative user scale (Figure 4), and (3) effect of collaboration rounds (Figure 5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "For VLMs: MME-P, MME-R, SEEDBench, MMBench, OCRBench, AI2D, MMMU-Val, MMMU-Dev. For LLMs: MT-Bench (1st/2nd turn), AlpacaEval 2.0 (LC Win, Win), Arena-Hard."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The LLM benchmarks (AlpacaEval 2.0, Arena-Hard, MT-Bench) use LLM-as-judge evaluation, not human evaluation. No human evaluation of the system's outputs is performed. Given the paper claims about response quality improvement, human evaluation would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "All benchmarks used (MME, SEED Bench, MMBench, MMMU, AlpacaEval, Arena-Hard, MT-Bench) are standard test sets with established evaluation protocols. There is no tuning on these sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-benchmark breakdowns for VLMs across 8 benchmarks. Table 2 provides per-benchmark breakdowns for LLMs. Table 3 provides per-client ablation breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses diminishing returns with increasing collaboration rounds (Figure 5), performance drops in some metrics for some models (e.g., Qwen2.5-VL-7B* shows drops in MME-P and OCRBench in Table 1), and notes that using only LLaVA as a collaborator can hurt performance on some benchmarks (Table 3, negative deltas)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows some metrics decrease for certain models (e.g., Qwen2.5-VL-7B* drops on MME-P from 1693.53 to 1656.04, and OCRBench from 881 to 865). Table 3 shows collaboration with only LLaVA hurts Janus-Pro-7B on most benchmarks. The appendix Table 4 shows Qwen2.5-Coder-1.5B* drops significantly on MT-Bench."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims CoLM 'consistently improves model performance on previously failed queries.' The results in Tables 1-2 generally support consistent improvements across most benchmarks and models, though some individual metrics show decreases."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims that collaboration improves performance. The ablation studies (removing individual client models, varying number of clients, varying rounds) provide controlled manipulations that support these causal claims. Each ablation changes a single variable."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Collaborative Large Models via A Client-Server Paradigm' and abstract claim broad applicability, but experiments are limited to specific model combinations (GPT-4o as server, specific 7B client models). The paper does not bound its claims to these specific configurations. The conclusion section acknowledges limitations about specialized client model availability but frames it as a future opportunity rather than a scope boundary."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements. For example, improvements could be partly due to the strong GPT-4o server model simply overriding weaker client answers rather than genuine 'collaboration.' The paper does not consider whether the gains are primarily from the server model's quality rather than the collaborative framework itself."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o' without specifying an API version or snapshot date. Other models are named (Qwen2.5-VL-7B-Instruct, Janus-Pro-7B, LLaVA-v1.5-7B) with some specificity, but GPT-4o is used as both a server model and client model without a version identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompts in natural language (e.g., 'You are an expert in math' as a specialization prompt, and prompts that 'encourage consistency, factual accuracy') but never provides the actual full prompt text used in experiments. The prompts for the server aggregation step and the client refinement step are described only at a high level."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other inference hyperparameters are reported for any of the models used. The paper uses multiple LLM APIs without stating sampling settings."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-stage client-server pipeline for LLMs and the two-step process for VLMs are described in the Method section: (1) client models generate responses independently, (2) server synthesizes, (3) clients refine. The VLM variant concatenates responses as contextual input. The workflow is depicted in Figures 2 and 6."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how benchmark data was preprocessed, how images were handled for VLM inputs, or how queries were formatted for different models. The paper goes directly from naming the benchmarks to reporting results."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion contains two sentences acknowledging that specialized client-side models 'are not yet widely deployed,' but this is a brief mention, not a substantive discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The brief limitation mention in the conclusion is about future deployment scenarios, not about specific threats to the validity of the current experimental results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no discussion of which model types, tasks, or deployment scenarios are excluded from the claims. The framing is uniformly positive about broad applicability."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-example predictions, model outputs, intermediate responses) is made available. Only aggregated benchmark scores are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection is straightforward: all benchmarks used are named and cited (MME, SEED Bench, MMBench, AI2D, OCRBench, MMMU, AlpacaEval 2.0, Arena-Hard, MT-Bench). The models used are listed with their sources (OpenAI API, Hugging Face)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. All data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from query input through client generation, server aggregation, and client refinement is described at a high level, but the actual data processing pipeline (how benchmark queries are fed in, how responses are collected and scored) is not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. The authors are affiliated with China Telecom's AI institute (TeleAI), Northwestern Polytechnical University, and the University of Hong Kong, but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Institute of Artificial Intelligence (TeleAI) at China Telecom, Northwestern Polytechnical University, and The University of Hong Kong."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The first author affiliation is with TeleAI (China Telecom's AI institute). Since no funding is disclosed, independence cannot be assessed. The paper does not evaluate China Telecom's own products, but the corporate affiliation and lack of funding disclosure means independence cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates pre-trained models (GPT-4o, Qwen, DeepSeek, LLaVA) on benchmarks but does not state any training data cutoff dates for these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark data could have been in the training data of any of the models used. Several benchmarks (e.g., MME, MMMU) are well-known and could be in training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Many benchmarks used (e.g., SEED Bench 2023, MMBench 2024, MT-Bench 2023) were published before the models' likely training cutoffs. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved. This is a benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper proposes a method that makes multiple API calls (client models generate, server aggregates, clients refine) but reports no API costs, token counts, or latency measurements. This is especially relevant since the paper argues for 'efficient collaboration' and a 'deployment-friendly framework.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total API spend, hardware specifications, or total computation time are reported. The paper claims practical deployment benefits but provides no quantification of the computational overhead."
       }
     }
   },
   "claims": [
     {
       "claim": "CoLM consistently improves model performance on previously failed queries through collaborative client-server reasoning.",
       "evidence": "Tables 1 and 2 show improvements across most benchmarks for both VLMs and LLMs. Table 1 shows average score improvements for all tested VLMs (e.g., Janus-Pro-7B from 47.55 to 58.06, LLaVA-1.5-7B from 38.62 to 56.13). Table 2 shows LLM improvements across MT-Bench, AlpacaEval 2.0, and Arena-Hard.",
       "supported": "moderate"
     },
     {
       "claim": "Weaker models benefit more from collaboration than stronger models.",
       "evidence": "Table 1 shows larger absolute gains for weaker models: LLaVA-1.5-7B gains +17.51 average score points vs. GPT-4o gaining +1.02. The pattern holds for LLMs in Table 2 where initially weak models (e.g., Deepseek-Math-7B) show larger improvements.",
       "supported": "moderate"
     },
     {
       "claim": "Full collaboration with all client models outperforms collaboration with any single client model.",
       "evidence": "Table 3 shows that Janus-Pro-7B* (all models collaborating) achieves the best performance across all benchmarks compared to single-model collaboration variants.",
       "supported": "moderate"
     },
     {
       "claim": "Increasing the number of collaborative users consistently improves performance.",
       "evidence": "Figure 4 shows performance increasing with more clients across three LLM benchmarks, though with diminishing returns.",
       "supported": "moderate"
     },
     {
       "claim": "CoLM is practical for real-world deployment under client-server architectures.",
       "evidence": "The paper provides theoretical motivation for the client-server paradigm alignment but no empirical evidence of deployment practicality (no latency, cost, or scalability measurements).",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CoLM proposes a client-server paradigm for collaborative LLM/VLM reasoning where lightweight client models generate initial responses, a central server model (GPT-4o) synthesizes them, and clients refine based on server guidance. Experiments across 8 VLM benchmarks and 3 LLM benchmarks show consistent improvements, especially for weaker models (e.g., LLaVA-1.5-7B average score from 38.62 to 56.13). Ablation studies demonstrate that full multi-model collaboration outperforms single-model collaboration, with diminishing returns as more clients or rounds are added.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single-run point estimates with no confidence intervals, error bars, standard deviations, or significance tests. Given the stochastic nature of LLM inference (especially with API-based models like GPT-4o), the reported improvements could be within noise ranges."
     },
     {
       "flag": "Missing cost analysis for a cost-efficiency-motivated method",
       "detail": "The paper motivates CoLM partly on deployment efficiency but reports zero cost metrics. The method requires multiple model inferences per query (all client models + server model + client refinement), which could be substantially more expensive than a single model call. Without cost data, the practical deployment claim is unsupported."
     },
     {
       "flag": "Strong server model confound",
       "detail": "GPT-4o serves as the server model. Improvements to weaker client models (7B parameter models) could largely be attributed to GPT-4o's superior capabilities simply overriding poor answers, rather than genuine collaborative synergy. The paper does not control for this by testing with a weaker server model."
     },
     {
       "flag": "No reproducibility materials",
       "detail": "No code, prompts, hyperparameters, or environment specifications are provided. The method description is at a high level, making independent reproduction difficult."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section. Only two sentences in the conclusion briefly acknowledge the availability of specialized client models as a limitation."
     }
   ],
   "cited_papers": [
     {
       "title": "Mixture-of-Agents Enhances Large Language Model Capabilities",
       "authors": ["J. Wang", "J. Wang", "B. Athiwaratkun", "C. Zhang", "J. Zou"],
       "year": 2024,
       "arxiv_id": "2406.04692",
       "relevance": "Directly compared baseline (MoA) for multi-model collaborative LLM reasoning."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2024,
       "relevance": "Cascading LLM inference method for cost-efficient model selection, related to deployment efficiency claims."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs with Preference Data",
       "authors": ["I. Ong", "A. Almahairi", "V. Wu", "W.-L. Chiang", "T. Wu", "J. E. Gonzalez", "M. W. Kadous", "I. Stoica"],
       "year": 2024,
       "arxiv_id": "2406.18665",
       "relevance": "Dynamic model routing for cost-efficient LLM inference, related approach to multi-model deployment."
     },
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Y. Du", "S. Li", "A. Torralba", "J. B. Tenenbaum", "I. Mordatch"],
       "year": 2023,
       "relevance": "Multi-agent debate framework for improving LLM reasoning through model collaboration."
     },
     {
       "title": "Rethinking Mixture-of-Agents: Is Mixing Different Large Language Models Beneficial?",
       "authors": ["W. Li", "Y. Lin", "M. Xia", "C. Jin"],
       "year": 2025,
       "arxiv_id": "2502.00674",
       "relevance": "Critical analysis of multi-model mixture approaches for LLMs, directly relevant to evaluating collaborative model paradigms."
     },
     {
       "title": "Think Deep, Think Fast: Investigating Efficiency of Verifier-free Inference-time-scaling Methods",
       "authors": ["J. Wang", "S. Zhu", "J. Saad-Falcon", "B. Athiwaratkun", "Q. Wu", "J. Wang", "S. L. Song", "C. Zhang", "B. Dhingra", "J. Zou"],
       "year": 2025,
       "arxiv_id": "2504.14047",
       "relevance": "Studies inference-time scaling methods for LLMs, relevant to understanding computational efficiency of collaborative approaches."
     },
     {
       "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
       "authors": ["L. Zheng", "W.-L. Chiang", "Y. Sheng", "S. Zhuang", "Z. Wu", "Y. Zhuang", "Z. Lin", "Z. Li", "D. Li", "E. P. Xing", "H. Zhang", "J. E. Gonzalez", "I. Stoica"],
       "year": 2023,
       "arxiv_id": "2306.05685",
       "relevance": "Defines MT-Bench and LLM-as-judge evaluation methodology used as a primary benchmark in this paper."
     },
     {
       "title": "Length-Controlled AlpacaEval: A Simple Way to Debias Automatic Evaluators",
       "authors": ["Y. Dubois", "B. Galambosi", "P. Liang", "T. B. Hashimoto"],
       "year": 2024,
       "arxiv_id": "2404.04475",
       "relevance": "AlpacaEval 2.0 benchmark with length-controlled debiasing, used as a primary LLM evaluation benchmark."
     },
     {
       "title": "Eagle: Efficient training-free router for multi-LLM inference",
       "authors": ["Z. Zhao", "S. Jin", "Z. M. Mao"],
       "year": 2024,
       "arxiv_id": "2409.15518",
       "relevance": "Training-free multi-LLM routing approach, related to model selection in multi-model inference systems."
     }
   ]
 }