ai-research-survey

scan.json (25638B)

---

 {
   "paper": {
     "title": "Collab: Controlled Decoding using Mixture of Agents for LLM Alignment",
     "authors": [
       "Souradip Chakraborty",
       "Sujay Bhatt",
       "Udari Madhushani Sehwag",
       "Alec Koppel",
       "Soumya Suvra Ghosal",
       "Jiahao Qiu",
       "Mengdi Wang",
       "Dinesh Manocha",
       "Furong Huang",
       "Sumitra Ganesh"
     ],
     "year": 2025,
     "venue": "ICLR 2025",
     "arxiv_id": "2503.21720"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive link is provided anywhere in the paper. The paper mentions 'Reproducibility is ensured through the use of publicly available resources' (Section 5), but this refers to the datasets and models used, not their own code release."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: Berkeley Nectar (Zhu et al., 2023) and HH-RLHF (Bai et al., 2022), as detailed in Table 3. All models used are open-source (Zephyr-7B, Starling-7B, etc.)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A states 'We run all experiments with Python 3.7.4 and PyTorch 1.9.0. For all experimentation, we use two Nvidia RTX A6000 GPUs.' This mentions Python and PyTorch versions plus GPU type, but there is no requirements.txt, Dockerfile, or detailed dependency list sufficient to recreate the full environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. There is no README, no scripts to replicate experiments, and no repository. The paper describes the algorithm (Algorithm 1) but does not provide instructions for replicating specific experimental results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The results in Figure 2 (average rewards), Figure 3 (diversity and coherence), and Table 1 (GPT-4 win-tie rates) are all presented as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Collab 'consistently outperforms' baselines and achieves 'up to 1.56x improvement' but no statistical significance tests (p-values, t-tests, etc.) are reported to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'achieving an improvement of up to 1.56x in average reward and 71.89% in GPT-4 based win-tie rate.' Table 1 provides specific win-tie percentages for each evaluation setup, and Figure 2 shows normalized average rewards with baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '300 prompts' were sampled for GPT-4 evaluation but does not justify this sample size. The number of test prompts for the average reward evaluations is not specified, and no power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any results. The figures show single-run bar charts without error bars. There is no mention of averaging across multiple runs or seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against individual agent decoding (Agent-I, Agent-II using SoTA decoding from Chakraborty et al., 2024b) and Best-of-N (BoN) sampling (Nakano et al., 2021), as described in Section 5 and shown in Figures 2-3 and Table 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines use state-of-the-art controlled decoding methods (TQ* from Chakraborty et al., 2024b; ARGS from Khanov et al., 2024), which are contemporaneous with this work. The individual agents use SoTA decoding approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4a shows an ablation comparing 'Switch without Diversity' (two similar models) vs. 'Switch with Diversity' (two diverse agents), demonstrating the importance of agent diversity. Figure 4b shows the effect of increasing the number of diverse agents."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports four metrics: average reward (Figure 2), GPT-4 win-tie rate (Table 1), diversity (Figure 3), and coherence (Figure 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. GPT-4 is used as a 'surrogate for human assessment' (Section 5), but this is an automated evaluation, not actual human judgment. The paper evaluates alignment quality, where human evaluation of outputs would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper refers to a 'test set' and 'test prompts' (Section 5), and the evaluation is conducted on prompts from established benchmark datasets (Berkeley Nectar, HH-RLHF) which have standard train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across 7 evaluation setups (Evaluation 1-7) spanning two tasks (Task-I and Task-II), as shown in Table 1 and Figures 2-3, with different model combinations per setup as detailed in Table 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss failure cases or where the approach breaks down. Examples in Appendix G show outputs from different strategies but do not highlight failures of the proposed method. Example-3 (the chicken math problem) shows Collab getting the correct answer (54), but there is no systematic error analysis."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows improvement for Collab over baselines. No configurations or settings that failed or hurt performance are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '1.56x in average reward' and '71.89% in GPT-4 based win-tie rate.' Table 1 shows win-tie rates, with Evaluation-2 for Collab vs Agent-I at 71.89%. Figure 2 shows normalized reward improvements consistent with the 1.56x claim. The theoretical claims about optimality bounds are proven in Theorem 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (Figure 4a: diversity improves performance; Figure 4b: more agents improve performance) and controlled comparisons (same prompts, same datasets, varying only the decoding strategy). The ablation design involves controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper's title and claims are framed broadly as 'LLM Alignment' but experiments are conducted only with 7B-parameter open-source models on two datasets (Berkeley Nectar and HH-RLHF). The paper does not explicitly state boundaries on generalization to larger models, other task types, or different model families."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements. For example, it does not address whether improvements could be attributed to effectively having more compute at inference time (evaluating multiple models), or whether the reward model used for evaluation biases toward certain response styles."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 lists model names like 'Zephyr-7B-α', 'Starling-7B-α', 'Mistral-7B-α-IT', 'Dolphin-2.6-Mistral-7B-DPO', etc. While these are specific model names, no Hugging Face model IDs, snapshot dates, or version hashes are provided. GPT-4 is used for evaluation but no version (e.g., gpt-4-0613) is specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The GPT-4 evaluation prompt is described in natural language ('We prompt GPT-4 to rate responses from various decoding strategies on relevance, accuracy, and insightfulness, scoring them from 1 to 10') but the actual prompt text is not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states 'we set the number of tokens sampled (top-p) p = 10 and the decoding alignment parameter α = 1.' Maximum prompt length (128) and continuation length (2048) are specified. Greedy-based sampling is stated for baselines."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding in the sense of tools, retry logic, or feedback mechanisms. The term 'agents' here refers to different LLM policies combined at the token decoding level, not an agentic system with tools."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how prompts were selected from Berkeley Nectar or HH-RLHF datasets, how many prompts were used for the main evaluations (only 300 mentioned for GPT-4 evaluation), or any filtering/preprocessing steps applied to the datasets."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The paper has an Introduction, Related Work, Problem Formulation, Theoretical Analysis, Experimental Evaluations, Acknowledgments, Disclaimer, and Appendices, but no limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no discussion of potential issues with the evaluation methodology, model selection, or generalizability of results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do not show. There is no discussion of limitations regarding model size (only 7B tested), dataset coverage, or situations where the approach might not work."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (generated responses, reward scores, per-prompt results) is made available. Only aggregated metrics are shown in figures and tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes the data sources: Berkeley Nectar dataset for Task-I (multi-turn dialogues and question answering) and HH-RLHF dataset for Task-II (helpfulness and ethical alignment). Table 3 details the specific model combinations for each evaluation setup."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study uses standard public benchmark datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw dataset prompts to final evaluation results is not fully documented. The paper does not describe how test prompts were selected, how the 300 prompts for GPT-4 evaluation were sampled, or intermediate processing steps."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section lists specific funding sources: DARPA TIAMAT 80321, NSF-IIS-2147276, DOD-ONR N00014-22-1-2335, DOD-AFOSR FA9550-23-1-0048, DOD-DARPA GARD HR00112020007, Adobe, Capital One, and JP Morgan faculty fellowships."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: JPMorgan AI Research, University of Maryland College Park, and Princeton University. The paper header identifies the first author's work as 'done as a part of an internship at JPMorgan AI Research.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (DARPA, NSF, DOD, Adobe, Capital One, JP Morgan) do not have a direct financial stake in the specific outcome of this alignment method comparison. JPMorgan AI Research is a co-affiliation but the paper evaluates open-source models and publicly available datasets, not JPMorgan products."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. There is a JPMorgan disclaimer about informational purposes but no explicit declaration of personal financial interests (patents, equity, etc.)."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses pre-trained models (Zephyr-7B, Starling-7B, Mistral-7B, Dolphin models) but does not state their training data cutoff dates. The evaluation benchmarks (Berkeley Nectar, HH-RLHF) could potentially overlap with model training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation prompts from Berkeley Nectar or HH-RLHF may have been seen during the training of the models used (Zephyr, Starling, Dolphin, Mistral). HH-RLHF in particular is widely used and likely in many models' training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HH-RLHF (Bai et al., 2022) and Berkeley Nectar (Zhu et al., 2023) are public datasets published before many of the models used were trained. The paper does not address whether these benchmarks were in the models' training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The method requires running multiple LLMs and evaluating implicit Q-functions at each token, which is substantially more expensive than single-agent decoding. No inference cost, latency, or tokens-per-second measurements are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A mentions 'two Nvidia RTX A6000 GPUs' but does not state total GPU hours, wall-clock time for experiments, or any cost estimate for running the multi-agent decoding approach."
       }
     }
   },
   "claims": [
     {
       "claim": "Collab achieves up to 1.56x improvement in average reward over single-agent decoding baselines.",
       "evidence": "Figure 2 shows normalized average reward values across 6 evaluation setups, with Collab consistently outperforming Agent-I, Agent-II, and BoN sampling baselines.",
       "supported": "moderate"
     },
     {
       "claim": "Collab achieves 71.89% GPT-4 based win-tie rate against the best individual agent.",
       "evidence": "Table 1 shows GPT-4 evaluation results. Collab vs Agent-I on Task-I Evaluation-2 achieves 71.89% win-tie rate. Results range from 50.00% to 73.75% across setups.",
       "supported": "moderate"
     },
     {
       "claim": "The sub-optimality of the multi-agent decoding policy is bounded by the reward difference between the target and best model plus KL divergence terms.",
       "evidence": "Theorem 1 (Section 4) with full proof in Appendix D.1 establishes the sub-optimality bound. The proof uses Lemma 1 and properties of KL-regularized RL.",
       "supported": "strong"
     },
     {
       "claim": "Diversity among agents enhances collaborative performance in the mixture.",
       "evidence": "Figure 4a compares 'Switch without Diversity' vs 'Switch with Diversity', showing improved average reward with diverse agents. Figure 4b shows increasing reward with more diverse agents.",
       "supported": "moderate"
     },
     {
       "claim": "Collab outperforms baselines in diversity and coherence metrics.",
       "evidence": "Figure 3 shows diversity and coherence values for Evaluations 1-3, with Collab outperforming other baselines in both metrics.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "Collab proposes a mixture-of-agents controlled decoding strategy that dynamically selects the most suitable LLM at each token position using an implicit Q-function as the guiding metric. The approach provides theoretical sub-optimality bounds showing performance improves when the best agent's reward function is close to the target. Empirical evaluations on 7 setups using 7B open-source models show consistent improvements over single-agent decoding baselines, with up to 1.56x average reward improvement and 71.89% GPT-4 win-tie rate. Agent diversity is shown to be important for collaborative performance gains.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are reported as point estimates without confidence intervals, error bars, or variance across runs. For a method involving stochastic decoding from multiple models, the lack of any variance reporting is a significant gap — it is unclear whether the improvements are within noise."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Comparative claims ('outperforms', 'consistently outperforms') are made solely by comparing raw numbers without any statistical tests. Some win-tie rates are as low as 50.00% (Evaluation 5 vs BoN), suggesting Collab does not always meaningfully outperform baselines."
     },
     {
       "flag": "No limitations section",
       "detail": "A published ICLR paper with no limitations, threats to validity, or scope boundaries discussion. The method's practical computational overhead (running multiple models per token) is never acknowledged or quantified."
     },
     {
       "flag": "Inference cost not reported",
       "detail": "The method requires sampling top-p tokens from each of K agents and evaluating the implicit Q-function for each, making it substantially more expensive than single-agent decoding. This cost is never quantified, making practical applicability difficult to assess."
     },
     {
       "flag": "GPT-4 as sole quality evaluator",
       "detail": "GPT-4 is used as a 'surrogate for human assessment' but no actual human evaluation is conducted. GPT-4 evaluation has known biases (position bias, verbosity preference, self-preference) which are not discussed."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "HH-RLHF (2022) and Berkeley Nectar (2023) are public datasets that likely appeared in the training data of the models evaluated (Zephyr, Starling, Dolphin, Mistral). This potential contamination is never discussed."
     },
     {
       "flag": "Limited model scale",
       "detail": "All experiments use 7B-parameter models only, but conclusions are framed broadly for 'LLM Alignment' without acknowledging this restriction."
     }
   ],
   "cited_papers": [
     {
       "title": "Controlled decoding from language models",
       "authors": ["Sidharth Mudgal", "Jong Lee", "Harish Ganapathy"],
       "year": 2024,
       "relevance": "Foundational work on inference-time alignment via controlled decoding that this paper builds upon and extends to multi-agent settings."
     },
     {
       "title": "Transfer Q Star: Principled Decoding for LLM Alignment",
       "authors": ["Souradip Chakraborty", "Soumya Suvra Ghosal", "Ming Yin"],
       "year": 2024,
       "arxiv_id": "2405.20495",
       "relevance": "Prior single-agent principled decoding method that serves as a key baseline and theoretical predecessor to Collab."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell"],
       "year": 2023,
       "relevance": "Foundational alignment method (DPO) that Collab aims to complement or replace through inference-time decoding rather than fine-tuning."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Seminal RLHF paper establishing the training-based alignment paradigm that decoding methods like Collab aim to avoid."
     },
     {
       "title": "Learning to decode collaboratively with multiple language models",
       "authors": ["Zejiang Shen", "Hunter Lang", "Bailin Wang"],
       "year": 2024,
       "doi": "10.18653/v1/2024.acl-long.701",
       "relevance": "Concurrent work on collaborative multi-model decoding, directly relevant to multi-agent LLM collaboration approaches."
     },
     {
       "title": "ARGS: Alignment as Reward-Guided Search",
       "authors": ["Maxim Khanov", "Jirayu Burapacheep", "Yixuan Li"],
       "year": 2024,
       "relevance": "State-of-the-art decoding alignment baseline used in the paper's experimental comparisons."
     },
     {
       "title": "DEAL: Decoding-time alignment for large language models",
       "authors": ["James Y Huang", "Sailik Sengupta", "Daniele Bonadiman"],
       "year": 2024,
       "relevance": "Alternative approach to inference-time LLM alignment that reframes text generation as a search problem."
     },
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Yuntao Bai", "Andy Jones", "Kamal Ndousse"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "Source of the HH-RLHF dataset used for evaluation and foundational work on safety alignment."
     },
     {
       "title": "Collaborative decoding of critical tokens for boosting factuality of large language models",
       "authors": ["Lifeng Jin", "Baolin Peng", "Linfeng Song"],
       "year": 2024,
       "arxiv_id": "2402.17982",
       "relevance": "Related work on collaborative multi-model decoding focused on factuality, relevant to multi-agent LLM coordination."
     },
     {
       "title": "Tuning language models by proxy",
       "authors": ["Alisa Liu", "Xiaochuang Han", "Yizhong Wang"],
       "year": 2024,
       "arxiv_id": "2401.08565",
       "relevance": "Alternative approach to leveraging multiple models for alignment without direct fine-tuning."
     },
     {
       "title": "Scaling laws for reward model overoptimization",
       "authors": ["Leo Gao", "John Schulman", "Jacob Hilton"],
       "year": 2023,
       "arxiv_id": "2210.10760",
       "relevance": "Documents reward overoptimization in RLHF, a key motivation for inference-time alignment approaches like Collab."
     }
   ]
 }