ai-research-survey

scan.json (25304B)

---

 {
   "paper": {
     "title": "OPTIMA: Optimizing Effectiveness and Efficiency for LLM-Based Multi-Agent System",
     "authors": ["Weize Chen", "Jiarui Yuan", "Chen Qian", "Cheng Yang", "Zhiyuan Liu", "Maosong Sun"],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2410.08115",
     "doi": "10.48550/arXiv.2410.08115"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "OPTIMA, an iterative generate-rank-select-train framework for LLM-based multi-agent systems, achieves up to 2.8x performance gain with less than 10% of tokens on information-exchange tasks using Llama 3 8B/3.2 3B. The framework integrates MCTS-inspired DPO data generation and demonstrates cross-task transfer (e.g., MATH to GSM8k). OPTIMA's token efficiency enables improved inference-time scaling laws, matching CoT self-consistency performance with 88.5% fewer tokens on GSM8k.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: HotpotQA, 2WikiMultiHopQA, TriviaQA, CBT, GSM8K, MATH, ARC-C, MMLU. All are standard public datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 8 A100 GPUs and Llama 3 8B / 3.2 3B but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions or scripts are provided. The appendix contains pseudo-code and hyperparameters but not executable reproduction instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, 6, 7 report only point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims OPTIMA 'outperforms' baselines but provides no statistical significance tests — comparisons are based solely on comparing numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes in context: '38.3% improvement (2.8x)' on 2WMHQA, '90% reduction in token usage', '88.5% fewer tokens'. Baseline numbers are provided alongside for context (Table 1)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the size of the evaluation sets is provided, nor any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across seeds, or spread measures are reported in any results table. All appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against CoT, Self-Consistency (n=8), Multi-Agent Debate (MAD), and AutoForm (Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include AutoForm (Chen et al., 2024c) and MAD (Du et al., 2024), which are contemporary multi-agent approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 presents ablation on reward components (removing #Tokens regularization and LM Loss), and the paper compares three OPTIMA variants (iSFT, iDPO, iSFT-DPO) which serve as ablations of the training strategy."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both task performance (F1/accuracy) and token count (#Tok) across all experiments. Different tasks use F1 and accuracy as appropriate."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via F1/accuracy metrics. Given claims about communication readability and quality, human evaluation would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses standard benchmark test sets (HotpotQA, GSM8K, MATH, etc.) which are separate from training data. Transfer experiments (Table 2) evaluate on entirely different datasets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-task breakdown across 8 benchmarks. Results are shown separately for information exchange and debate settings."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where OPTIMA underperforms: 'on MATH and GSM8k, OPTIMA variants show comparable or slightly lower performance than SC' (Section 3.1). Transfer limitations are also discussed: 'transferring to more distant domains remains challenging, e.g., we find it hard to transfer from MATH to ARC-C' (Section 3.2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that iDPO sometimes has performance trade-offs despite token reductions, that transfer from MATH to ARC-C fails, and that removing LM loss leads to unreadable outputs (Table 3, Section 3.4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'up to 2.8x performance gain with less than 10% tokens' which is supported by Table 1 (2WMHQA: iSFT-DPO F1 74.2 vs MAD 25.9, tokens 54.9 vs 543.7). Other claims about inference-time scaling are supported by Figure 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims via ablation studies (Table 3: removing reward components shows their individual effects). The iterative training progression (Figure 1) and controlled ablations constitute adequate causal evidence for component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Optimizing Effectiveness and Efficiency for LLM-Based Multi-Agent System' broadly, but results are on Llama 3 8B/3.2 3B only in two-agent scenarios on specific QA/reasoning benchmarks. The paper mentions this limitation but the title and abstract frame it as a general MAS solution."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for the improvements. For example, it does not consider whether the gains come primarily from the format diversity initialization rather than the iterative training, or whether the improvements are specific to the task-model combinations tested."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes its measurements (F1, accuracy, token count) from broader claims. It explicitly discusses the relationship between token efficiency and compute utilization, and frames benchmark scores appropriately as task-specific metrics."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'Llama 3 8B / 3.2 3B' (Meta, 2024) as the base model. These are specific model sizes. GPT-4 is mentioned for prompt pool generation (Table 12) without version, but GPT-4 is not the evaluated model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Tables 9-13 in the appendix, including prompts for information exchange, GSM8k/MATH, ARC-C/MMLU, format pool generation, and a format pool example."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 provides comprehensive hyperparameters for all OPTIMA variants across all 8 tasks, including learning rates, epochs, batch sizes, λ values, β, α, and threshold parameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent communication scaffold is described in detail: two-agent conversation with turn-based communication, format specification prompts, solver/critic roles for debate tasks. Algorithms 1-5 provide pseudo-code for the full pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes how multi-hop datasets are split between agents ('we split relevant contexts between two agents'), how TriviaQA/CBT contexts are randomly assigned, and how the format pool was created (Table 12). The data selection pipeline (generate, rank, select with thresholds) is detailed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "There is a dedicated 'Limitations' section after the conclusion with substantive discussion of multiple limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific threats: 'experiments primarily focus on two-agent scenarios with a shared model architecture', 'transferring OPTIMA-trained models to substantially different application areas... remains unexplored', and 'real-world deployment scenarios may involve additional constraints'."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states what was NOT tested: scaling to 5-10 agents, heterogeneous agent configurations, cross-domain transfer (e.g., QA to coding), and real-world deployment constraints."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, generated trajectories, reward scores) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data generation process is thoroughly described: trajectory sampling with N samples per instance, reward computation, selection criteria with thresholds, and MCTS-based DPO data generation (Sections 2.2-2.5, Algorithms 1-5)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper uses standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: initialization with format pool → trajectory generation → reward ranking → top-70% selection → training → iteration. Each step is formalized with equations and pseudo-code."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is visible in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Tsinghua University and Beijing University of Posts and Telecommunications. No commercial product is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama 3 8B / 3.2 3B but does not state the training data cutoff date for these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the Llama 3 models were pre-trained on any of the benchmark datasets (HotpotQA, GSM8K, MATH, etc. are all public and could appear in training data)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All benchmarks used (HotpotQA 2018, GSM8K 2021, MATH 2021, ARC, MMLU) were published well before Llama 3's training. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Token counts are reported for all methods across all tasks in Table 1, and inference token scaling is analyzed in Figure 3 and Section 3.3."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states: 'Iterative training completes within 12 hours on 8 A100 GPUs for most tasks, except MATH, which takes around 24 hours.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. All results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs producing the reported results is not stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Table 8 reports final hyperparameters but no information about how they were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the hyperparameters in Table 8 were selected. The paper does not discuss whether selection was on validation or test data."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied despite comparing multiple methods across 8 benchmarks."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement OPTIMA and all baselines (MAD, AutoForm) but do not acknowledge the bias of evaluating their own system against their own baseline implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 (right panel) explicitly plots performance as a function of token usage (compute proxy) for various methods, and Section 3.3 analyzes inference-time scaling laws."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the selected benchmarks actually measure multi-agent collaboration effectiveness or whether simpler explanations (e.g., better single-agent fine-tuning) could account for the results."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "OPTIMA changes both the training procedure and the communication scaffold simultaneously. The paper does not isolate whether the gains come from the training or from the structured communication formats introduced during initialization."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All benchmarks (HotpotQA 2018, MATH 2021, GSM8K 2021, etc.) predate Llama 3 (2024). No discussion of temporal leakage risk."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "In the information exchange setting, contexts are split between agents. No discussion of whether this splitting could leak information (e.g., partial answers visible in context)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of train/test independence. Training data for OPTIMA is generated from the same benchmark distributions used for evaluation."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "OPTIMA achieves up to 2.8x performance gain with less than 10% of tokens on information-asymmetric QA tasks.",
       "evidence": "Table 1: On 2WMHQA, iSFT-DPO achieves F1 74.2 vs MAD's 25.9 (2.8x), using 54.9 tokens vs MAD's 543.7 (10.1%). Section 3.1.",
       "supported": "strong"
     },
     {
       "claim": "OPTIMA-trained models transfer effectively across tasks within the same domain.",
       "evidence": "Table 2: Models trained on HotpotQA transfer to 2WMHQA (iSFT F1 56.5 vs MAD 25.9) and TriviaQA. MATH-trained iDPO achieves 77.9% on GSM8k vs direct MAD 72.5%. Section 3.2.",
       "supported": "strong"
     },
     {
       "claim": "OPTIMA improves inference-time scaling laws, matching CoT-SC with 88.5% fewer tokens.",
       "evidence": "Figure 3 right panel: GSM8k-iDPO matches CoT-SC accuracy with 88.5% fewer tokens. Section 3.3.",
       "supported": "moderate"
     },
     {
       "claim": "The reward function components (token count, LM loss) each contribute meaningfully to the framework.",
       "evidence": "Table 3: Removing token count increases tokens 2.7-6.3x; removing LM loss decreases performance by up to 10.7 F1 points. Section 3.4.",
       "supported": "strong"
     },
     {
       "claim": "OPTIMA scales to three-agent scenarios.",
       "evidence": "Table 7 in Appendix G shows results for three agents, but performance is worse than two-agent settings for IE tasks. Limited to one task per setting.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or statistical tests",
       "detail": "All results are reported as point estimates across 8 benchmarks and 7+ methods. No error bars, standard deviations, or significance tests are provided. Without knowing variance, it is impossible to assess whether differences are meaningful."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "All benchmarks predate Llama 3's training data. The base model may have memorized benchmark answers, which could interact differently with the OPTIMA training than with baselines. This is especially concerning since OPTIMA fine-tunes on benchmark-specific data."
     },
     {
       "flag": "No code release",
       "detail": "Despite extensive algorithmic detail, no code is released. The framework involves many implementation decisions (MCTS parameters, edit distance thresholds, etc.) that are hard to reproduce from pseudo-code alone."
     },
     {
       "flag": "Self-comparison bias",
       "detail": "The authors implement all baselines (MAD, AutoForm) themselves. Lucic et al. (2018) showed that authors' baseline implementations systematically underperform. No independent evaluation or use of official baseline implementations is mentioned."
     }
   ],
   "cited_papers": [
     {
       "title": "Beyond natural language: LLMs leveraging alternative formats for enhanced reasoning and communication",
       "authors": ["Weize Chen", "Chenfei Yuan", "Jiarui Yuan"],
       "year": 2024,
       "arxiv_id": "2402.18439",
       "relevance": "AutoForm baseline; explores non-natural-language communication formats for LLM agents."
     },
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Yilun Du", "Shuang Li", "Antonio Torralba"],
       "year": 2024,
       "relevance": "Multi-Agent Debate (MAD) baseline; foundational work on LLM multi-agent debate."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell"],
       "year": 2023,
       "relevance": "DPO algorithm used as a core training method in OPTIMA."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2023,
       "relevance": "Self-consistency baseline and inference-time scaling technique used in evaluation."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": ["Chen Qian"],
       "year": 2024,
       "relevance": "LLM-based multi-agent system for software development with role-based communication."
     },
     {
       "title": "AgentVerse: Facilitating multi-agent collaboration and exploring emergent behaviors",
       "authors": ["Weize Chen"],
       "year": 2024,
       "relevance": "Multi-agent collaboration framework with role-playing for reasoning."
     },
     {
       "title": "An empirical analysis of compute-optimal inference for problem-solving with language models",
       "authors": ["Yangzhen Wu", "Zhiqing Sun"],
       "year": 2024,
       "arxiv_id": "2408.00724",
       "relevance": "Inference-time compute scaling analysis directly relevant to OPTIMA's scaling claims."
     },
     {
       "title": "Large language monkeys: Scaling inference compute with repeated sampling",
       "authors": ["Bradley Brown"],
       "year": 2024,
       "arxiv_id": "2407.21787",
       "relevance": "Inference-time scaling via repeated sampling, directly compared to OPTIMA's efficiency approach."
     },
     {
       "title": "Iterative reasoning preference optimization",
       "authors": ["Richard Yuanzhe Pang", "Weizhe Yuan"],
       "year": 2024,
       "arxiv_id": "2404.19733",
       "relevance": "Iterative DPO/RPO method that OPTIMA builds upon for its iDPO variant."
     },
     {
       "title": "Scaling large-language-model-based multi-agent collaboration",
       "authors": ["Chen Qian", "Zihao Xie"],
       "year": 2024,
       "arxiv_id": "2406.07155",
       "relevance": "Addresses MAS scaling challenges that OPTIMA aims to solve."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong"],
       "year": 2024,
       "relevance": "Prominent multi-agent framework for software development."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversation framework",
       "authors": ["Qingyun Wu"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Multi-agent conversation framework widely used for LLM-based MAS."
     }
   ]
 }