ai-research-survey

scan.json (20544B)

---

 {
   "paper": {
     "title": "Cross-Modal Memory Compression for Efficient Multi-Agent Debate",
     "authors": ["Jing Wu", "Yue Sun", "Tianpei Xie", "Suiyao Chen", "Jingyuan Bao", "Yaopengxiao Xu", "Gaoyuan Du", "Inseok Heo", "Alexander Gutfraind", "Xin Wang"],
     "year": 2025,
     "venue": "ICML 2025 (submission)"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. Appendix B describes implementation details but no link to source code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: GSM8K, MATH, and GPQA. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B mentions AdamW optimizer, learning rate 1e-4, batch size 64, and adapter dimensions, but no requirements.txt, Dockerfile, or library versions are provided. Hardware is mentioned only as 'single NVIDIA A100 GPU' for inference timing."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided. Appendix B and C describe configuration details but not how to reproduce results end-to-end."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-3 are reported as point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims DebateOCR outperforms baselines across models and datasets but provides no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., '92.4% and 75.0% reductions' in tokens, '2.25× and 2.38× speedups', and accuracy differences between methods in Table 1."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why these three benchmarks or their standard test set sizes are sufficient. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or multi-run results are reported. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines are compared: T-MAD (text-based multi-agent debate) and TS-MAD (text-based with summarization), described in Section 5.1.2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines represent current standard approaches to multi-agent debate (Du et al. 2023, Khan et al. 2024, Chen et al. 2024a, Liu et al. 2025). These are contemporary methods."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 provides ablation studies: image resolution vs. accuracy/tokens (Table 2) and comparison of vision encoders (Table 3). Appendix D provides scaling analysis across agent counts and debate rounds."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: accuracy, token consumption, and inference time (Section 5.1.3, Table 1)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates automated reasoning benchmarks with exact-match accuracy. Human evaluation is not relevant to these claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard test splits are used: GSM8K test set (1,319 problems), MATH test set (5,000 problems), GPQA (448 questions). These are separate from training data described in Appendix B."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (4 models) and per dataset (3 datasets) in Table 1, and per resolution in Table 2."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes that on GPQA with InternVL-8B, summarization slightly outperforms DebateOCR (23.3% vs 22.8%) and discusses why: 'extractive summaries better retain domain-specific terminology for graduate-level questions.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The GPQA/InternVL case where DebateOCR underperforms is reported. Table 2 shows lower resolutions degrade accuracy (224x224 yields 71.2% vs 76.3% at 1024x1024)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims >92% token reduction and substantially lower compute cost — Table 1 shows 92.4% reduction on InternVL/GSM8K and consistent reductions across all settings. Accuracy claims are supported by Table 1 results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims via ablation studies (removing/varying components like resolution in Table 2, encoder choice in Table 3). The discussion in Section 5.2 explains accuracy gains through artifact removal, supported by the theoretical framework."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Multi-Agent Debate' generally but results are only on mathematical/scientific reasoning with 4 open-source VLMs in the 7-12B range. No evaluation on non-STEM tasks, closed-source models, or larger models. The conclusion claims 'visual representations offer a practical and efficient alternative to text-based communication in multi-agent systems' without bounding to the tested domain."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper offers one explanation for accuracy gains (artifact removal via information bottleneck theory) but does not consider alternatives such as: the vision encoder acting as a regularizer, the adapter's training data distribution effects, or whether majority voting dynamics change with compressed inputs."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'Qwen2.5-VL-7B-Instruct', 'Llama-3.2-11B-Vision', 'InternVL-8B', 'Pixtral-12B'. While some include size, none specify exact checkpoint versions, snapshot dates, or commit hashes."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides full prompt templates for all three prompt strategies (vision-augmented, pure text, text+summary) with actual template text and mechanism descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B reports learning rate (1e-4), batch size (64), adapter dimensions (dh=d=4096), rendering resolution (1024x1024), font size (12), line spacing (1.2). Section 5.1 reports K=3 agents, R=5 rounds, max 1024 tokens per response."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent debate framework is described in detail: Sections 3.3-3.4 cover the SAM-CLIP-adapter pipeline, text-to-image rendering, vision encoding, and context injection. The debate protocol (majority voting, round structure) is specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B documents training data sources (~85,000 samples from GSM8K, MATH, MathQA, MMLU-STEM, SQuAD, NaturalQuestions) and rendering configuration. Test set sizes and evaluation procedures are specified in Section 5.1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The conclusion generalizes broadly to 'multi-agent systems' without bounding to mathematical reasoning or the tested model sizes."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-example predictions, debate traces, intermediate outputs) is made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Training data sources are listed in Appendix B. Evaluation uses standard public benchmarks with specified test splits (Section 5.1.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from text rendering to vision encoding to context injection is documented in Sections 3.3-3.4 and Appendix B. Training procedure and evaluation pipeline are described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "Author affiliations are not listed in the paper text (likely anonymized for review)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four models used. The adapter is trained on public datasets but the base models' training data provenance is not discussed."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GSM8K, MATH, or GPQA test examples could have been in the base models' training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021) and MATH (2021) are widely known benchmarks that could be in training data of models released in 2024-2025. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports inference time per sample in seconds for all methods. Token consumption is reported in thousands. Section 5.1.3 defines how these are measured."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training compute for the adapter is not quantified (no GPU hours, training time, or total cost). Only inference is measured on 'a single NVIDIA A100 GPU.'"
       }
     }
   },
   "claims": [
     {
       "claim": "DebateOCR cuts input tokens by more than 92% compared to text-based multi-agent debate.",
       "evidence": "Table 1 shows token reduction from 59.2K to 4.5K on InternVL-8B/GSM8K (92.4% reduction). Similar reductions across all model-dataset combinations.",
       "supported": "strong"
     },
     {
       "claim": "DebateOCR achieves the best or competitive accuracy across most settings while dramatically reducing tokens.",
       "evidence": "Table 1 shows DebateOCR achieves highest accuracy in 11 of 12 model-dataset combinations. One exception: GPQA with InternVL-8B where TS-MAD achieves 23.3% vs 22.8%.",
       "supported": "strong"
     },
     {
       "claim": "Visual compression achieves 2.25× inference speedup over text-based debate.",
       "evidence": "Table 1 shows inference time reductions, e.g., InternVL-8B/GSM8K: 64.7s vs 145.7s (2.25×). Consistent across models.",
       "supported": "strong"
     },
     {
       "claim": "Multi-agent aggregation enables compressed histories to approach the information bottleneck with exponentially high probability.",
       "evidence": "Theorem 4.2 with full proof in Appendix A. The theoretical result relies on assumptions (A.3-A.5) that are discussed but not rigorously validated empirically.",
       "supported": "moderate"
     },
     {
       "claim": "The method generalizes effectively across diverse MLLMs and scales linearly with debate rounds and agent count.",
       "evidence": "Table 1 shows results across 4 models. Appendix D shows scaling analysis across 2-8 agents and 1-8 rounds. However, all models are open-source VLMs in the 7-12B range.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DebateOCR replaces textual debate histories with compact image representations in multi-agent debate systems, achieving >92% token reduction with competitive or improved accuracy across GSM8K, MATH, and GPQA benchmarks using four open-source VLMs (7-12B parameters). The method provides 2.25× inference speedup over text-based debate. A theoretical analysis based on information bottleneck theory explains why compression can maintain accuracy through multi-agent diversity. Ablation studies show 1024×1024 resolution offers the best accuracy-efficiency tradeoff.",
   "red_flags": [
     {
       "flag": "No error bars or multi-run variance",
       "detail": "All results appear to be single-run numbers with no standard deviation, confidence intervals, or repeated trials. Multi-agent debate outcomes can vary with sampling, so this is a significant omission."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section. Known limitations such as restriction to mathematical reasoning, small open-source models, and potential contamination are not discussed."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "GSM8K and MATH were published in 2021 and are widely used. Models released in 2024-2025 may have trained on these benchmarks. This risk is not discussed, and it could affect all methods equally or differentially depending on how compression interacts with memorized solutions."
     },
     {
       "flag": "Overbroad generalization claims",
       "detail": "The conclusion claims the approach works for 'multi-agent systems' generally, but evaluation is limited to mathematical/scientific reasoning with 7-12B open-source VLMs. No evaluation on coding, creative, or language tasks."
     }
   ],
   "cited_papers": [
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Y. Du", "S. Li", "A. Torralba", "J. B. Tenenbaum", "I. Mordatch"],
       "year": 2023,
       "relevance": "Foundational multi-agent debate paper that DebateOCR builds upon and compares against."
     },
     {
       "title": "Debating with more persuasive LLMs leads to more truthful answers",
       "authors": ["A. Khan", "J. Hughes", "D. Valentine", "L. Ruis", "K. Sachan"],
       "year": 2024,
       "arxiv_id": "2402.06782",
       "relevance": "Key multi-agent debate paper used as baseline approach (T-MAD)."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["M. Cemri", "M. Z. Pan", "S. Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analysis of failure modes in multi-agent LLM systems, directly relevant to survey scope."
     },
     {
       "title": "ChatEval: Towards better LLM-based evaluators through multi-agent debate",
       "authors": ["C.-M. Chan", "W. Chen", "Y. Su"],
       "year": 2023,
       "arxiv_id": "2308.07201",
       "relevance": "Multi-agent debate applied to LLM evaluation, relevant to agentic workflows."
     },
     {
       "title": "MetaGPT: Meta programming for multi-agent collaborative framework",
       "authors": ["S. Hong", "X. Zheng", "J. Chen"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent collaboration framework relevant to agentic AI systems."
     },
     {
       "title": "Encouraging divergent thinking in large language models through multi-agent debate",
       "authors": ["T. Liang", "Z. He", "W. Jiao"],
       "year": 2023,
       "arxiv_id": "2305.19118",
       "relevance": "Multi-agent debate methodology for improving LLM reasoning diversity."
     },
     {
       "title": "Reconcile: Round-table conference improves reasoning via consensus among diverse LLMs",
       "authors": ["J. Chen", "S. Saha", "M. Bansal"],
       "year": 2024,
       "relevance": "Multi-agent consensus approach for improving LLM reasoning, a baseline methodology."
     },
     {
       "title": "Let models speak ciphers: Multiagent debate through embeddings",
       "authors": ["C. Pham", "B. Liu", "Y. Yang"],
       "year": 2023,
       "arxiv_id": "2310.06272",
       "relevance": "Alternative approach to multi-agent debate communication using embeddings rather than text."
     },
     {
       "title": "Multiagent finetuning: Self improvement with diverse reasoning chains",
       "authors": ["V. Subramaniam", "Y. Du", "J. B. Tenenbaum"],
       "year": 2025,
       "arxiv_id": "2501.05707",
       "relevance": "Multi-agent approach to LLM self-improvement through diverse reasoning."
     },
     {
       "title": "Examining inter-consistency of large language models collaboration: An in-depth analysis via debate",
       "authors": ["K. Xiong", "X. Ding", "Y. Cao"],
       "year": 2023,
       "arxiv_id": "2305.11595",
       "relevance": "Analysis of LLM collaboration consistency in debate settings."
     }
   ]
 }