ai-research-survey

scan.json (21861B)

---

 {
   "paper": {
     "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
     "authors": [
       "Qingyun Wu",
       "Gagan Bansal",
       "Jieyu Zhang",
       "Yiran Wu",
       "Beibin Li",
       "Erkang Zhu",
       "Li Jiang",
       "Xiaoyun Zhang",
       "Shaokun Zhang",
       "Jiale Liu",
       "Ahmed Awadallah",
       "Ryen W. White",
       "Doug Burger",
       "Chi Wang"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2308.08155"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided: https://github.com/microsoft/autogen (footnote 2 in abstract)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Evaluations use publicly available benchmarks: MATH dataset, Natural Questions, ALFWorld. The 100-task OptiGuide dataset and 12-task group chat tasks are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section. Only mentions pre-installing 'sympy' for math experiments and using specific LLM models."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README with commands, or scripts to replicate experiments. The paper describes workflows conceptually but does not provide runnable reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Figures 4a-4d report only point estimates (e.g., '52.5%', '69.48%') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'AutoGen achieves the highest problem-solving success rate' and '15% performance gain' are made by comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements with baselines are provided, e.g., 'AutoGen achieves 69.48% vs GPT-4's 55.18%' (Section A1), '15% performance gain' (Section A3), 'multi-agent design boosts F-1 by 8% (GPT-4) and 35% (GPT-3.5)' (Section A4)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes. The 120 level-5 problems, 134 ALFWorld tasks, 100 OptiGuide tasks, and 12 group chat tasks are used without explanation of why these sizes are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations or variance across runs reported. The qualitative evaluation tests each problem 3 times but reports counts (e.g., '3/3', '2/3') without aggregate variance measures across the full evaluations."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines compared: AutoGPT, ChatGPT+Code Interpreter, ChatGPT+Plugin, vanilla GPT-4, Multi-Agent Debate, LangChain ReAct, DPR (Sections A1-A4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are contemporary (2023): ChatGPT+Code Interpreter, Multi-Agent Debate, LangChain ReAct, MetaGPT, CAMEL. These represent the state of the art at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Ablations conducted: interactive retrieval vs without (A2, Figure 4b), 2-agent vs 3-agent with grounding agent (A3, Figure 4c), single-agent vs multi-agent for safeguard (A4, Figure 4d), with vs without board agent (A6)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: success ratio for A1/A3, F1 and Recall for A2/A4. Different metrics appropriate to each task domain."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Qualitative evaluation with manual inspection in Table 2 (A1), where failure reasons are manually analyzed. User experience analysis comparing verbosity and undesired behaviors across systems."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "ALFWorld uses '134 unseen tasks' (Section A3). MATH uses the standard test dataset. Natural Questions uses existing evaluation splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "MATH results show only overall accuracy (69.48%) and level-5 subset (52.5%) but no per-category breakdown across the 6 categories mentioned. ALFWorld and other tasks also show only aggregate numbers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides detailed failure reasons for each system. Section A3 discusses 'occasional inability to leverage basic commonsense knowledge' and 'getting stuck in a loop.' A6 discusses illegitimate moves without board agent."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Reports that BabyAGI, CAMEL, and MetaGPT 'are not suitable choices for solving math problems out of the box' (Appendix D). Also notes AutoGen fails 1/3 on second math problem."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are general ('effectiveness of the framework in many example applications') and are supported by the six applications with empirical results in Section 3 and Appendix D."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'introducing a grounding agent could bring in a 15% performance gain' and 'multi-agent design boosts F-1 by 8%' are supported by controlled ablation studies (removing/adding specific components)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper title and abstract claim it is 'a generic framework for building diverse applications of various complexities' but tests on only 6 specific applications. Section 4 acknowledges 'early experimental stages' but the title/abstract are unbounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. For example, performance gains could be due to additional compute (more LLM calls) rather than the multi-agent design. The ethics statement and future work section do not address confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Paper says 'GPT-4' and 'GPT-3.5-turbo' without specific version snapshots (e.g., gpt-4-0613). The ALFWorld baseline uses 'text-davinci-003' which is a specific model but the main experiments lack version specificity."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 (Appendix C) provides the full default system message for the AssistantAgent. The grounding agent's commonsense prompt is quoted in Section A3. The role-play prompt for speaker selection is mentioned but described in less detail."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other LLM inference hyperparameters reported for any experiment."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent scaffolding is described in detail: Section 2 describes conversable agents, auto-reply mechanisms, conversation programming, GroupChatManager. Figure 2 illustrates the workflow. Each application (A1-A6) describes its specific agent topology."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "For MATH, '120 randomly selected level-5 problems' are used but the selection procedure is not documented. The 100 OptiGuide tasks are 'crafted to include equal numbers of safe and unsafe tasks' but crafting details are absent. The 12 group chat tasks are 'manually crafted' without criteria."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 (Discussion) and Appendix B.2 (Future Work) discuss limitations including safety challenges, difficulty of logging/adjusting complex workflows, and risks of fully autonomous agent conversations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Limitations are generic: 'fully autonomous agent conversations will need to be used with care,' 'high level of autonomy can also pose potential risks.' No specific threats to the validity of the experimental results are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries stated. The paper does not specify what the results do NOT show. Section 4 says 'this work is still in its early experimental stages' but does not state specific untested settings or claims not being made."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (agent conversation logs, individual problem results, timing data) is made available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Data collection for custom datasets (100 OptiGuide tasks, 12 group chat tasks) is not described in detail. For benchmark datasets, references are provided but selection criteria for subsets (e.g., 120 MATH problems) are not specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited for a study. The human-in-the-loop scenarios are demonstrations, not controlled user studies."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of the data pipeline from raw benchmark data to final reported results. For example, how the 120 problems were sampled, or how success/failure was determined for each system."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section mentions 'Qingyun Wu would like to acknowledge the funding and research support from the College of Information Science and Technology at Penn State University.' Microsoft Research affiliation is listed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Microsoft Research, Pennsylvania State University, University of Washington, Xidian University. The Microsoft affiliation is relevant since AutoGen is a Microsoft product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Most authors are from Microsoft Research, and AutoGen is a Microsoft open-source project (github.com/microsoft/autogen). Microsoft has a direct interest in demonstrating the effectiveness of its own framework."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement. Several authors are Microsoft employees evaluating a Microsoft product, but this conflict is not explicitly acknowledged beyond listing affiliations."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for GPT-4 or GPT-3.5-turbo despite evaluating them on benchmarks like MATH (published 2021) and Natural Questions."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4 or GPT-3.5-turbo may have seen MATH, Natural Questions, or ALFWorld problems during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MATH dataset (2021), Natural Questions (2019), and ALFWorld (2021) all predate GPT-4's likely training cutoff. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study conducted. Human-in-the-loop demonstrations are not controlled user studies."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study requiring IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or latency reported for any experiment despite making multiple LLM calls per problem across multiple systems."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or hardware specifications stated."
       }
     }
   },
   "claims": [
     {
       "claim": "AutoGen achieves 69.48% accuracy on the full MATH test dataset, outperforming vanilla GPT-4 at 55.18%.",
       "evidence": "Figure 4a and Appendix D report these numbers on the MATH dataset using GPT-4 as the base model.",
       "supported": "moderate"
     },
     {
       "claim": "AutoGen's interactive retrieval mechanism significantly improves QA performance over non-interactive retrieval.",
       "evidence": "Figure 4b shows F1 improvement from 15.12% to 25.88% and Recall from 58.56% to 66.65% on Natural Questions with GPT-3.5.",
       "supported": "moderate"
     },
     {
       "claim": "Introducing a grounding agent brings a 15% average performance gain on ALFWorld.",
       "evidence": "Figure 4c shows 3-agent system at 69% average vs 2-agent at 54% on 134 unseen ALFWorld tasks.",
       "supported": "moderate"
     },
     {
       "claim": "Multi-agent design boosts F-1 score for unsafe code detection by 8% (GPT-4) and 35% (GPT-3.5) over single-agent.",
       "evidence": "Figure 4d shows multi-agent vs single-agent F1 scores on 100 OptiGuide coding tasks.",
       "supported": "moderate"
     },
     {
       "claim": "AutoGen reduces OptiGuide core workflow code from 430 lines to 100 lines.",
       "evidence": "Section A4 states this claim but no code comparison or verification is provided.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "AutoGen is an open-source multi-agent conversation framework that enables building LLM applications through customizable, conversable agents and conversation programming. Empirical evaluations across six applications (math problem solving, RAG, decision-making, multi-agent coding, dynamic group chat, conversational chess) show performance improvements over baselines like vanilla GPT-4, LangChain ReAct, and Multi-Agent Debate. The framework demonstrates that multi-agent architectures with specialized agents (e.g., grounding agents, safeguard agents) can meaningfully improve task success rates compared to single-agent approaches.",
   "red_flags": [
     {
       "flag": "Vendor self-evaluation",
       "detail": "Most authors are Microsoft Research employees evaluating AutoGen, a Microsoft open-source project. This conflict of interest is not explicitly acknowledged beyond listing affiliations."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are reported as point estimates without confidence intervals, error bars, standard deviations, or significance tests. Claims of superiority rest entirely on raw number comparisons."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "GPT-4 is evaluated on MATH (2021), Natural Questions (2019), and ALFWorld (2021), all of which predate GPT-4's training. No contamination analysis is provided."
     },
     {
       "flag": "Small or ad-hoc evaluation sets",
       "detail": "Several evaluations use small or custom datasets: 12 manually crafted group chat tasks, qualitative evaluation on only 2 math problems tested 3 times each, 100 'crafted' OptiGuide tasks with no description of crafting methodology."
     },
     {
       "flag": "No cost comparison",
       "detail": "Multi-agent systems make more LLM calls than single-agent baselines, but no cost, latency, or token usage is reported. Performance gains may come at disproportionate computational cost."
     }
   ],
   "cited_papers": [
     {
       "title": "MetaGPT: Meta Programming for Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent framework for software development, direct competitor to AutoGen in multi-agent LLM applications."
     },
     {
       "title": "CAMEL: Communicative Agents for 'Mind' Exploration of Large Scale Language Model Society",
       "authors": ["Guohao Li"],
       "year": 2023,
       "relevance": "Pioneering multi-agent role-playing framework, compared with AutoGen in Table 1."
     },
     {
       "title": "Encouraging Divergent Thinking in Large Language Models through Multi-Agent Debate",
       "authors": ["Tian Liang"],
       "year": 2023,
       "relevance": "Multi-agent debate approach for improving LLM reasoning, used as baseline in AutoGen evaluations."
     },
     {
       "title": "Improving Factuality and Reasoning in Language Models through Multiagent Debate",
       "authors": ["Yilun Du"],
       "year": 2023,
       "arxiv_id": "2305.14325",
       "relevance": "Multi-agent debate for factuality improvement, foundational work for multi-agent LLM approaches."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational agent prompting technique combining reasoning and acting, used as baseline."
     },
     {
       "title": "Voyager: An Open-Ended Embodied Agent with Large Language Models",
       "authors": ["Guanzhi Wang"],
       "year": 2023,
       "arxiv_id": "2305.16291",
       "relevance": "LLM-based agent for open-ended exploration, relevant to agentic AI capabilities."
     },
     {
       "title": "Generative Agents: Interactive Simulacra of Human Behavior",
       "authors": ["Joon Sung Park"],
       "year": 2023,
       "arxiv_id": "2304.03442",
       "relevance": "Multi-agent simulation with LLM-powered agents, influential work in agentic AI."
     },
     {
       "title": "The Rise and Potential of Large Language Model Based Agents: A Survey",
       "authors": ["Zhiheng Xi"],
       "year": 2023,
       "arxiv_id": "2309.07864",
       "relevance": "Comprehensive survey of LLM-based agents, provides broader context for multi-agent frameworks."
     },
     {
       "title": "A Survey on Large Language Model Based Autonomous Agents",
       "authors": ["Lei Wang"],
       "year": 2023,
       "arxiv_id": "2308.11432",
       "relevance": "Survey of autonomous LLM agents, complementary to multi-agent conversation frameworks."
     },
     {
       "title": "Large Language Models as Tool Makers",
       "authors": ["Tianle Cai"],
       "year": 2023,
       "arxiv_id": "2305.17126",
       "relevance": "LLM tool-use capabilities relevant to agent frameworks and agentic AI."
     },
     {
       "title": "An Empirical Study on Challenging Math Problem Solving with GPT-4",
       "authors": ["Yiran Wu"],
       "year": 2023,
       "arxiv_id": "2306.01337",
       "relevance": "Empirical evaluation of GPT-4 on math problems, directly related to AutoGen's math evaluation."
     }
   ]
 }