ai-research-survey

scan.json (24252B)

---

 {
   "paper": {
     "title": "BAMAS: Structuring Budget-Aware Multi-Agent Systems",
     "authors": [
       "Liming Yang",
       "Junyu Luo",
       "Xuanzhe Liu",
       "Yiling Lou",
       "Zhenpeng Chen"
     ],
     "year": 2025,
     "venue": "AAAI 2026",
     "arxiv_id": "2511.21572",
     "doi": null
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We publicly release our code and data at https://github.com/chunfenri/BAMAS, to support further research' in the contributions section."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks used (GSM8K, MBPP, MATH) are all publicly available standard benchmarks. The paper also states data is released at their GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using PuLP library with CBC solver, all-MiniLM-L6-v2 for embeddings, Adam optimizer, and specific LLM providers, but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides algorithm pseudocode and hyperparameter tables but does not include step-by-step reproduction instructions or README-level commands to replicate the main experiments. The GitHub link is provided but the paper itself does not contain reproduction instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1, 2, and 4 are reported as single point estimates (e.g., '95.3%' accuracy, '542.9' cost) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims like 'BAMAS achieves comparable performance while reducing cost by up to 86%' but does not report any statistical significance tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., '62% cost reduction' (542.9 vs 1,425.3), '86% reduction' (529.2 vs 3,735.1), and accuracy comparisons like '95.3% vs. 95.4%'. These provide sufficient context for the reader to assess effect magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses standard benchmark sizes (GSM8K: 1,319 test, MBPP: 500 test, MATH: 1,000 test) but never justifies why these sample sizes are adequate for the claims being made. For MATH, a stratified sample of 1,000 is used but the justification is 'to manage its size,' not statistical adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using greedy decoding (temperature 0.0) for determinism and setting random seeds, but reports only single-run results. No standard deviation, variance, or spread across multiple runs is reported. The RL training uses a single seed per dataset."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against three state-of-the-art multi-agent construction approaches: AutoGen, MetaGPT, and ChatDev. Additionally, a Naive-CostAware heuristic baseline is introduced for ablation."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "AutoGen (2024), MetaGPT (2024), and ChatDev (2024) are all recent and widely cited multi-agent frameworks, representing the current state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 presents a component analysis comparing BAMAS against Naive-CostAware, which ablates both the ILP-based LLM provisioning and the RL-based topology selection components. Table 4 shows this comparison."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports two metrics: Accuracy (Acc %) for task performance and Average Cost (Avg Cost) for cost efficiency. Out-of-budget task counts are also reported in Table 3."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates on automated benchmarks (GSM8K, MBPP, MATH) where correctness is objectively verifiable. Human evaluation is not relevant for these tasks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For all three datasets, the paper clearly separates training data (used for RL training) from test data. GSM8K: first 1,000 training examples for RL, full 1,319 test set for evaluation. MBPP: 374 training problems for RL, 500 test problems. MATH: 1,000 stratified training, 1,000 stratified test."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides per-dataset breakdowns across three different benchmarks (Tables 1, 2), per-budget-level breakdowns, per-topology distributions across datasets and budgets (Figure 3), and out-of-budget counts per dataset/budget (Table 3)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses out-of-budget failures in Table 3 and notes that the Planner-Driven topology is never selected because 'its high cost and instability (since a poor plan can derail the entire process) make it a suboptimal choice.' The risk-averse behavior under low budgets is also discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the Planner-Driven topology is never selected despite offering the most flexibility. It also shows that at the lowest budget (500), BAMAS achieves lower accuracy than baselines (87.9% vs 95.4%), transparently showing the cost-accuracy trade-off."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'comparable performance while reducing cost by up to 86%.' This is supported by Table 1 on MBPP where BAMAS at budget 1,250 achieves 82.6% accuracy (comparable to MetaGPT's 82.2%) at cost 529.2 vs 3,735.1, an 86% reduction."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are primarily from ablation studies (RQ2). The comparison between BAMAS and Naive-CostAware, which removes the ILP provisioning and RL topology selection, constitutes a controlled single-variable manipulation showing the contribution of these components."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Structuring Budget-Aware Multi-Agent Systems' broadly, but results are tested on only three benchmarks (math word problems and basic Python programming) with only two LLMs (DeepSeek-V3 and GPT-4.1 nano). The paper does not bound its claims to these specific task types or LLM pairs. The conclusion states BAMAS 'achieves a state-of-the-art cost-performance trade-off' without qualification."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed results. For example, it does not consider whether the cost savings are primarily due to using cheaper models (GPT-4.1 nano) rather than the ILP/RL optimization itself, or whether the specific choice of only two LLMs constrains the generality of findings."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'DeepSeek-V3' and 'GPT-4.1 nano' with API names 'deepseek-chat' and 'gpt-4.1-nano' in Appendix B, but does not provide snapshot dates or specific API versions. These model behaviors can change across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes agent roles (Planner, Executor, Critic) and topology types but does not provide the actual prompts or system instructions used for the LLM agents. The prompts are described only conceptually."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides detailed hyperparameters: learning rates (0.0003 for GSM8K, 0.0015 for MATH/MBPP), Adam optimizer, batch size 20,000, 10 training epochs, entropy coefficient 0.001, temperature 0.0, max generation tokens per role per dataset (Table 7), and random seeds."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes four collaboration topologies (Linear, Star, Feedback, Planner-Driven) in detail, including role assignments, information flow, and coordination mechanisms. The generate-critique-revise loop for Feedback topology and the planner's dynamic orchestration are explained."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes data preprocessing: GSM8K uses first 1,000 training examples; MBPP uses all 374 training problems; MATH uses stratified sampling by difficulty level and problem type to construct 1,000-problem training and test sets. Cost estimation uses Tin=500 tokens with Tout determined by sampling 50 training instances."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The conclusion does not discuss any limitations either."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no consideration of potential weaknesses in the experimental design."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no discussion of which task types, LLM configurations, or deployment scenarios are outside the scope of the current evaluation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper states code and data are released at GitHub, but the paper itself does not describe making the raw experimental logs, per-example results, or RL training trajectories available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how benchmarks were obtained: GSM8K and MBPP use official training/test splits; MATH uses stratified sampling. RL training data collection is described via the topology selection process with offline experiences."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data comes from standard public benchmarks (GSM8K, MBPP, MATH)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: (1) benchmark selection and splitting, (2) cost estimation via token sampling, (3) ILP-based LLM provisioning, (4) offline RL training data collection with topology execution, (5) policy training, (6) evaluation on held-out test sets. The flow is clear from methodology through evaluation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgement section states: 'This work was supported by the National Natural Science Foundation of China under the grant number 62325201.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Peking University, UIUC, Nanyang Technological University, and Tsinghua University. None of the authors are affiliated with DeepSeek or OpenAI, the LLM providers evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is the National Natural Science Foundation of China, a government research funding agency with no commercial stake in the evaluated LLMs or the multi-agent framework."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial interests statement in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses DeepSeek-V3 and GPT-4.1 nano to solve benchmark problems (GSM8K, MBPP, MATH), but does not state the training data cutoff dates for either model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021), MBPP (2021), and MATH (2021) are all public benchmarks that predate the models used. The paper does not discuss whether these benchmarks may have been in the training data of DeepSeek-V3 or GPT-4.1 nano."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All three benchmarks (GSM8K, MBPP, MATH) were published in 2021 and are widely known. Both DeepSeek-V3 and GPT-4.1 nano were likely trained on data containing these benchmarks. The paper does not address this contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cost reporting is central to the paper. Average costs are reported per task across all experiments (e.g., BAMAS at budget 1,625 has average cost 542.9 on GSM8K). LLM pricing is provided (DeepSeek-V3: $0.27/$1.10 per million tokens in/out; GPT-4.1 nano: $0.10/$0.40)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While per-task LLM API costs are reported, the total computational budget for training the RL topology selection policy, running all experiments across budget levels and baselines, and the total API spend are not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "BAMAS achieves comparable performance while reducing cost by up to 86% compared to existing multi-agent construction approaches.",
       "evidence": "Table 1 shows BAMAS at budget 1,250 achieves 82.6% on MBPP vs MetaGPT's 82.2% at cost 529.2 vs 3,735.1, an 86% reduction. On GSM8K, BAMAS at budget 1,625 achieves 95.3% vs AutoGen's 95.4% at cost 542.9 vs 1,425.3 (62% reduction).",
       "supported": "strong"
     },
     {
       "claim": "BAMAS's joint optimization of LLM provisioning and topology selection significantly outperforms greedy baselines.",
       "evidence": "Table 4 (RQ2) shows BAMAS achieves peak 95.3% accuracy on GSM8K at cost 542.9 vs Naive-CostAware's peak 95.3% at cost 1,650.8. On MBPP, BAMAS achieves higher peak accuracy (82.6% vs 81.6%) at lower cost (529.2 vs 1,379.1).",
       "supported": "moderate"
     },
     {
       "claim": "BAMAS learns to adaptively select collaboration topologies based on task type and budget constraints.",
       "evidence": "Figure 3 (RQ3) shows Feedback topology dominates for math tasks (40.1% GSM8K, 69.8% MATH) while Linear dominates for code tasks (MBPP). Under tight budgets, simpler topologies (Linear, Star) are preferred. Planner-Driven is never selected.",
       "supported": "moderate"
     },
     {
       "claim": "BAMAS rarely exceeds the budget constraint during execution.",
       "evidence": "Table 3 shows 0 out-of-budget tasks on GSM8K across all budgets, 1-5 on MBPP, and at most 30/1,000 (3%) on MATH at budget 2,000.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "BAMAS combines Integer Linear Programming for budget-constrained LLM selection with reinforcement learning for collaboration topology selection to build cost-aware multi-agent systems. On three benchmarks (GSM8K, MBPP, MATH), BAMAS matches or exceeds state-of-the-art accuracy while reducing costs by up to 86% compared to AutoGen, MetaGPT, and ChatDev. The RL-trained topology selector learns task-specific preferences (Feedback for math, Linear for code) and exhibits risk-averse behavior under tight budgets by favoring simpler topologies.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats-to-validity, or scope-boundaries discussion whatsoever, which is concerning for a paper making broad claims about multi-agent system construction."
     },
     {
       "flag": "Only two LLMs tested",
       "detail": "BAMAS is evaluated with only two LLMs (DeepSeek-V3 and GPT-4.1 nano), yet the framework claims to solve the general problem of budget-aware multi-agent system structuring. It is unclear whether the ILP formulation and topology selection would behave similarly with a larger, more diverse LLM pool."
     },
     {
       "flag": "Benchmark contamination risk",
       "detail": "All three benchmarks (GSM8K, MBPP, MATH) are from 2021 and widely known. The models used (DeepSeek-V3, GPT-4.1 nano) were likely trained on data containing these benchmarks, but the paper does not acknowledge or address this risk."
     },
     {
       "flag": "Single-run results with no variance",
       "detail": "Despite using RL training with random seeds and non-deterministic LLM API calls, results are reported from single runs without error bars, confidence intervals, or variance across seeds."
     },
     {
       "flag": "No prompts provided",
       "detail": "The actual prompts given to LLM agents (Planner, Executor, Critic roles) are not provided, making it impossible to reproduce the exact system behavior even with the released code."
     }
   ],
   "cited_papers": [
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang", "Yiran Wu", "Shaokun Zhang", "Erkang Zhu", "Beibin Li", "Li Jiang", "Xiaoyun Zhang", "Chi Wang"],
       "year": 2024,
       "relevance": "Major multi-agent framework used as baseline; directly relevant to agentic AI system evaluation."
     },
     {
       "title": "MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong"],
       "year": 2024,
       "relevance": "Multi-agent system framework using meta-programming paradigm, used as baseline comparison."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["Chen Qian"],
       "year": 2024,
       "relevance": "Multi-agent software development system simulating a virtual company, used as baseline."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2024,
       "relevance": "Directly addresses cost-efficient LLM usage, a core concern of the survey."
     },
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["Aman Madaan"],
       "year": 2023,
       "relevance": "Foundational work on iterative self-refinement in LLM agents, relevant to agentic workflow evaluation."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2023,
       "relevance": "Key paper on LLM agent learning through verbal feedback, relevant to agent capability evaluation."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao"],
       "year": 2023,
       "relevance": "Seminal work on combining reasoning and action in LLM agents, foundational to agentic AI."
     },
     {
       "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
       "authors": ["Shunyu Yao"],
       "year": 2023,
       "relevance": "Important prompting/reasoning framework for LLMs, relevant to agent capability evaluation."
     },
     {
       "title": "Large Language Model-Based Agents for Software Engineering: A Survey",
       "authors": ["Junjielong Liu", "Kaixin Wang", "Yanjie Chen", "Xinrun Peng", "Zhenpeng Chen", "Lingming Zhang", "Yiling Lou"],
       "year": 2025,
       "relevance": "Survey of LLM agents for SE, directly in scope for the agentic AI research landscape."
     },
     {
       "title": "A Survey on LLM-Based Multi-Agent System: Recent Advances and New Frontiers in Application",
       "authors": ["Shuaihang Chen"],
       "year": 2025,
       "relevance": "Comprehensive survey of LLM multi-agent systems, relevant to understanding the broader field."
     },
     {
       "title": "Rethinking the Bounds of LLM Reasoning: Are Multi-Agent Discussions the Key?",
       "authors": ["Qineng Wang"],
       "year": 2024,
       "relevance": "Evaluates whether multi-agent discussions improve LLM reasoning, relevant to agent capability claims."
     },
     {
       "title": "A Unified Approach to Routing and Cascading for LLMs",
       "authors": ["Jasper Dekoninck", "Maximilian Baader", "Martin Vechev"],
       "year": 2025,
       "relevance": "Addresses cost-efficient LLM routing, closely related to budget-aware LLM selection."
     },
     {
       "title": "Efficient Contextual LLM Cascades through Budget-Constrained Policy Learning",
       "authors": ["Xiao Zhang"],
       "year": 2024,
       "relevance": "Budget-constrained policy learning for LLM cascades, directly relevant to cost-efficient LLM deployment."
     }
   ]
 }