ai-research-survey

scan.json (29922B)

---

 {
   "paper": {
     "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
     "authors": [
       "Qingyun Wu",
       "Gagan Bansal",
       "Jieyu Zhang",
       "Yiran Wu",
       "Beibin Li",
       "Erkang Zhu",
       "Li Jiang",
       "Xiaoyun Zhang",
       "Shaokun Zhang",
       "Jiale Liu",
       "Ahmed Awadallah",
       "Ryen W. White",
       "Doug Burger",
       "Chi Wang"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2308.08155"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/microsoft/autogen in the abstract (footnote 2). This is a working, public repository."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluations use publicly available benchmarks: MATH dataset (Hendrycks et al., 2021), Natural Questions (Kwiatkowski et al., 2019), ALFWorld (Shridhar et al., 2021), and MiniWoB++ (Shi et al., 2017). No proprietary datasets were created that would need separate release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. The paper mentions pre-installing 'sympy' for math experiments but does not provide comprehensive dependency or environment specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released and the paper describes workflows conceptually, there are no step-by-step reproduction instructions, scripts to replicate specific experiments, or a 'Reproducing Results' section. The paper describes configurations at a high level but does not provide specific commands or scripts to reproduce the benchmark results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Figures 4a-4d report only point estimates (e.g., '52.5%', '69.48%') with no confidence intervals or error bars. The OptiGuide Table 4 reports standard deviations for user interaction savings, but the primary benchmark results lack uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AutoGen outperforms baselines (e.g., 'the highest problem-solving success rate among all compared methods') but provides no statistical significance tests. All comparisons are based on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context: e.g., '15% performance gain on average' for grounding agent (Section A3), 'multi-agent design boosts the F-1 score by 8% (with GPT-4) and 35% (with GPT-3.5-turbo)' (Section A4), and '3x saving on user's time' (Section A4). Results include specific percentages with baselines visible in figures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for sample sizes. The MATH evaluation uses '120 randomly selected level-5 problems' without justification for this number. The qualitative evaluation uses only 2 problems tested 3 times each. The dynamic group chat pilot study uses 12 manually crafted tasks. The OptiGuide user study involves only a single expert participant. None of these sample sizes are justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The ALFWorld results report 'average' and 'best of 3' across 3 attempts, but no standard deviation or variance is reported for the main benchmark results (MATH, Natural Questions, ALFWorld, MiniWoB++). Table 4 reports standard deviations for the OptiGuide saving ratio, but this is a secondary metric. The primary success rates lack variance reporting."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included across applications: vanilla GPT-4, ChatGPT+Code Interpreter, ChatGPT+Plugin, LangChain ReAct, Multi-Agent Debate for A1; DPR for A2; ReAct for A3; single-agent vs multi-agent for A4; RCI for A7 (MiniWoB++)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are contemporary for 2023: ChatGPT+Code Interpreter, GPT-4 (released 2023), Multi-Agent Debate (Liang et al., 2023), MetaGPT (Hong et al., 2023), RCI (Kim et al., 2023). These represent the state of the art at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Several ablation studies are conducted: interactive retrieval vs. non-interactive in A2 (Figure 4b); two-agent vs. three-agent (with grounding agent) in A3 (Table 3); single-agent vs. multi-agent in A4 (Figure 4d); with vs. without board agent in A6; role-play vs. task-based speaker selection in A5 (Tables 5, 6)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: success ratio for MATH (A1); F1 and Recall for Q&A (A2, Figure 4b); success ratio per category for ALFWorld (A3, Table 3); F1 and Recall for OptiGuide (A4, Figure 4d); number of successes, LLM calls, and termination failures for dynamic group chat (A5, Tables 5-6)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "A user study comparing ChatGPT+Code Interpreter vs. AutoGen-based OptiGuide is conducted in A4 with an expert Python programmer, measuring time and accuracy on 10 questions. The qualitative evaluations in A1 also involve manual assessment of failure reasons."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The MATH evaluation uses the standard test dataset (5000 problems). ALFWorld uses '134 unseen tasks.' MiniWoB++ uses all available tasks from the official RCI code. These are standard held-out evaluation sets from published benchmarks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "ALFWorld results (Table 3) provide per-category breakdown across Pick, Clean, Heat, Cool, Look, and Pick 2 task types. MiniWoB++ (Figure 18) shows per-task success rates for all 49 tasks."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases are extensively discussed: Table 2 provides qualitative failure analysis for math problems; Figure 10 shows a failure trajectory in ALFWorld; Table 7 provides case analysis of typical failure tasks in MiniWoB++; the paper discusses loops and common error patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Negative results are reported: the two-agent system for ALFWorld does not outperform ReAct (both at 54% average); MiniWobChat achieves 52.8% which is 3.6% lower than RCI; preliminary evaluations show BabyAGI, CAMEL, and MetaGPT are 'not suitable choices for solving math problems out of the box.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims AutoGen agents are 'customizable, conversable' (demonstrated through framework design in Section 2), can 'operate in various modes' (shown through six applications in Section 3), and 'empirical studies demonstrate the effectiveness' (supported by benchmark results in Figures 4a-4d). Claims are appropriately hedged."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are primarily made through ablation studies which constitute controlled single-variable manipulations: adding/removing grounding agent (A3), interactive vs. non-interactive retrieval (A2), single vs. multi-agent (A4), with vs. without board agent (A6). These are adequate designs for the causal claims being made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim AutoGen is a framework for 'diverse applications of various complexities and LLM capacities,' but empirical results are limited to specific benchmarks (MATH, Natural Questions, ALFWorld, MiniWoB++, OptiGuide, 12 manually crafted tasks, and chess). The paper does not explicitly bound its generalization claims to these tested domains. The framework claims are broader than what is empirically tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for observed performance gains. For example, the improvement from the grounding agent in ALFWorld could be due to additional prompt information rather than the multi-agent architecture. The OptiGuide user study with a single expert does not discuss alternative explanations for time savings. No threats-to-validity section exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4' and 'GPT-3.5-turbo' throughout without specifying snapshot dates or API versions (e.g., 'gpt-4-0613'). The only specific version mentioned is 'text-davinci-003' for ReAct (from the official code, not the authors' choice). No model version identifiers are provided for the AutoGen experiments."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The default system message for the AssistantAgent is provided in full in Figure 5 (Appendix C). The retrieval-augmented chat prompt is shown in Figure 8. The ALFWorld few-shot prompts are referenced as obtained from the ReAct repository. While not every prompt is shown, the primary system prompts are provided in full."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other LLM inference hyperparameters are reported anywhere in the paper. The paper mentions configuring agents but does not specify the inference settings used for experiments."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent scaffolding is described in detail: Section 2 explains the conversable agent design, auto-reply mechanisms, conversation programming paradigm, and control flow. Each application section describes the specific agent topology, roles, and interaction patterns (e.g., Commander/Writer/Safeguard in A4, grounding agent in A3)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is documented for key experiments: MATH uses '120 randomly selected level-5 problems' from 6 categories excluding geometry; Natural Questions uses '5,332 non-redundant context documents and 6,775 queries from HuggingFace'; ALFWorld uses '134 unseen tasks'; OptiGuide uses '100 coding tasks crafted to include equal numbers of safe and unsafe tasks.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. The Discussion (Section 4) mentions future directions and the Ethics Statement discusses potential concerns, but neither constitutes a substantive limitations discussion addressing the empirical evaluations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential confounds in its evaluations, small sample sizes in qualitative studies, or the single-participant user study. The Ethics Statement discusses general ethical considerations but not study-specific validity threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'this work is still in its early experimental stages' (Section 4) but does not explicitly state what the results do NOT show. It does not bound claims to specific domains, model types, or task complexities tested. The Future Work section (Appendix B.2) raises open questions but does not state specific scope limitations of the current results."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental outputs (model responses, conversation logs, intermediate results) are not available for independent verification. Only selected example outputs are shown in Appendix E. There is no data repository or supplementary data files."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection procedures are described: MATH uses a random selection from the standard dataset; Natural Questions uses the HuggingFace version; ALFWorld uses the standard 134 unseen tasks; OptiGuide uses a crafted dataset of 100 tasks. The A5 pilot study uses '12 manually crafted complex tasks' with examples provided."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The OptiGuide user study (A4) mentions 'an expert Python programmer with proficiency in Gurobi participated in the test' but provides no recruitment details — how they were selected, their relationship to the project, or potential selection bias. This is relevant since the single participant's expertise could bias the results."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The data pipeline from raw experimental runs to reported results is not fully documented. For example, the MATH evaluation does not explain how the 120 level-5 problems were randomly selected, what random seed was used, or whether any runs were excluded. The MiniWoB++ evaluation references 'all available tasks in the official RCI code' without detailing which tasks those are."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section states: 'Qingyun Wu would like to acknowledge the funding and research support from the College of Information Science and Technology at Penn State University.' Microsoft Research affiliation is listed for the majority of authors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Microsoft Research, Pennsylvania State University, University of Washington, and Xidian University. The Microsoft affiliation is prominent given that AutoGen is a Microsoft Research project."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The majority of authors are from Microsoft Research, and AutoGen is a Microsoft Research project (hosted at github.com/microsoft/autogen). Microsoft has a direct financial and reputational interest in the success of AutoGen. The funder (Microsoft) is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. There is no declaration of financial interests, patents, or equity related to the work. Given that AutoGen is a Microsoft product, this disclosure would be relevant."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4 and GPT-3.5-turbo on benchmarks like MATH (published 2021) and Natural Questions (published 2019) but does not state the training data cutoff date for any model used. The only hint is mentioning that FLAML Spark APIs added in December 2022 are 'not encompassed in the GPT-4 training data,' but no explicit cutoff date is stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. MATH (2021), Natural Questions (2019), and ALFWorld (2021) are all public benchmarks that could have been in GPT-4's training data. This is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmarks used (MATH, Natural Questions, ALFWorld, MiniWoB++) were all published before GPT-4's likely training cutoff. The paper does not discuss contamination risk despite this being a known issue for these benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The OptiGuide user study (A4) involving a human participant is not pre-registered. No pre-registration link is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The OptiGuide user study involving a human participant does not mention IRB or ethics board approval."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The single user study participant is described only as 'an expert Python programmer with proficiency in Gurobi.' No further demographics (experience level, years of experience, relationship to the project, etc.) are reported."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for the user study participant are stated. The participant was simply described as 'an expert Python programmer with proficiency in Gurobi.'"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The user study is not an experimental study with multiple conditions assigned to different participants; the same participant used both systems. Randomization of task order is not described but this is more of an observational comparison than an RCT."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Blinding is not feasible in this user study since the participant interacts directly with both systems (ChatGPT+Code Interpreter vs. OptiGuide) and would necessarily know which system they are using."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "With only a single participant, attrition is not explicitly discussed. The paper does not state whether additional participants were planned or whether the study was designed for just one participant."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs or token consumption is reported. The paper notes 'approximately 19.4% of questions trigger an Update Context operation, resulting in additional LLM calls' (A2) and reports LLM call counts in Table 6, but does not translate these into dollar costs or token counts."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, total API spend, or hardware specifications are reported. The evaluations involve multiple GPT-4 API calls across thousands of benchmark examples, but the total cost is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "AutoGen achieves 69.48% overall accuracy on the full MATH test dataset, compared to 55.18% for vanilla GPT-4.",
       "evidence": "Section A1 (Appendix D) reports quantitative evaluation on the entire MATH test set (5000 problems) with results shown in Figure 4a.",
       "supported": "moderate"
     },
     {
       "claim": "AutoGen with interactive retrieval outperforms DPR and AutoGen without interactive retrieval on the Natural Questions QA task.",
       "evidence": "Figure 4b shows F1 and Recall metrics: AutoGen (25.88% F1, 66.65% Recall) vs. DPR (22.79% F1, 62.59% Recall) vs. AutoGen W/O interactive retrieval (15.12% F1, 58.56% Recall). An ablation confirms the interactive retrieval mechanism's role.",
       "supported": "moderate"
     },
     {
       "claim": "Adding a grounding agent to the ALFWorld system brings a 15% performance gain on average over the two-agent design.",
       "evidence": "Table 3 shows 3-agent system at 69% average success rate vs. 54% for 2-agent system across 134 unseen ALFWorld tasks. Figure 10 provides a qualitative case study.",
       "supported": "moderate"
     },
     {
       "claim": "Multi-agent design boosts F1 score in identifying unsafe code by 8% (GPT-4) and 35% (GPT-3.5-turbo) compared to single-agent.",
       "evidence": "Figure 4d shows the comparison on 100 coding tasks (50 safe, 50 unsafe) in the OptiGuide application.",
       "supported": "moderate"
     },
     {
       "claim": "AutoGen-based OptiGuide saves around 3x of user's time and reduces user interactions by 3-5 times compared to ChatGPT+Code Interpreter.",
       "evidence": "Section A4 reports a user study with a single expert participant on 10 questions, with time measurements (4min 35sec vs 1.5min). Table 4 reports interaction saving ratios across 5 OptiGuide applications.",
       "supported": "weak"
     },
     {
       "claim": "AutoGen's core workflow code for OptiGuide was reduced from over 430 lines to 100 lines.",
       "evidence": "Stated in Section A4, but the comparison methodology (what constitutes 'core workflow code') is not rigorously defined.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "AutoGen is an open-source multi-agent conversation framework from Microsoft Research that introduces conversable agents and conversation programming paradigms. Across six diverse applications (math problem solving, retrieval-augmented QA, text-world decision making, multi-agent coding, dynamic group chat, and conversational chess), the framework demonstrates competitive or superior performance compared to existing approaches. Key empirical results include 69.48% accuracy on MATH (vs. 55.18% for vanilla GPT-4), a 15% success rate improvement on ALFWorld from adding a grounding agent, and competitive performance on MiniWoB++ compared to the specialized RCI method.",
   "red_flags": [
     {
       "flag": "Vendor self-evaluation",
       "detail": "The majority of authors are from Microsoft Research, and AutoGen is a Microsoft Research project. The paper evaluates AutoGen favorably against competitors without acknowledging this conflict of interest or including independent evaluation."
     },
     {
       "flag": "Single-participant user study",
       "detail": "The OptiGuide user study (A4) uses a single 'expert Python programmer with proficiency in Gurobi' to claim 3x time savings. This sample size is far too small for generalizable claims, and the participant's expertise and relationship to the project are not disclosed."
     },
     {
       "flag": "No uncertainty quantification on main results",
       "detail": "Primary benchmark results (MATH, Natural Questions, ALFWorld, MiniWoB++) are reported as single point estimates without confidence intervals, error bars, or significance tests. This makes it impossible to assess whether observed differences are meaningful."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "GPT-4 is evaluated on MATH (2021), Natural Questions (2019), and ALFWorld (2021), all published before GPT-4's training cutoff. The paper does not discuss whether these benchmarks may have been in the training data."
     },
     {
       "flag": "Small and manual pilot study for group chat",
       "detail": "The dynamic group chat (A5) evaluation uses only 12 manually crafted tasks. These tasks may be biased toward scenarios where AutoGen's group chat excels, and the sample size is too small for robust conclusions."
     },
     {
       "flag": "Missing model version specifications",
       "detail": "All experiments use 'GPT-4' and 'GPT-3.5-turbo' without snapshot dates or API versions. Given that model behavior changes across versions, results may not be reproducible."
     }
   ],
   "cited_papers": [
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Yilun Du", "Shuang Li", "Antonio Torralba", "Joshua B Tenenbaum", "Igor Mordatch"],
       "year": 2023,
       "arxiv_id": "2305.14325",
       "relevance": "Foundational work on multi-agent debate for improving LLM factuality, directly compared as a baseline in AutoGen's math evaluation."
     },
     {
       "title": "Encouraging divergent thinking in large language models through multi-agent debate",
       "authors": ["Tian Liang", "Zhiwei He", "Wenxiang Jiao"],
       "year": 2023,
       "relevance": "Multi-agent debate framework showing divergent thinking benefits, compared as a baseline in AutoGen."
     },
     {
       "title": "MetaGPT: Meta programming for multi-agent collaborative framework",
       "authors": ["Sirui Hong", "Xiawu Zheng", "Jonathan Chen"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Specialized multi-agent framework for software development, compared with AutoGen in Table 1 and preliminary math evaluations."
     },
     {
       "title": "CAMEL: Communicative agents for 'mind' exploration of large scale language model society",
       "authors": ["Guohao Li", "Hasan Abed Al Kader Hammoud", "Hani Itani"],
       "year": 2023,
       "relevance": "Multi-agent communicative framework using role-playing and inception prompting, compared with AutoGen in Table 1."
     },
     {
       "title": "Voyager: An open-ended embodied agent with large language models",
       "authors": ["Guanzhi Wang", "Yuqi Xie", "Yunfan Jiang"],
       "year": 2023,
       "arxiv_id": "2305.16291",
       "relevance": "Open-ended LLM agent for embodied tasks, relevant as a specialized single-agent approach for comparison."
     },
     {
       "title": "Generative agents: Interactive simulacra of human behavior",
       "authors": ["Joon Sung Park", "Joseph C O'Brien", "Carrie J Cai"],
       "year": 2023,
       "arxiv_id": "2304.03442",
       "relevance": "Multi-agent simulation of human behavior using LLMs, relevant to understanding agent interaction patterns."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Key prompting technique integrating reasoning and acting that is compared as a baseline in multiple AutoGen applications."
     },
     {
       "title": "The rise and potential of large language model based agents: A survey",
       "authors": ["Zhiheng Xi", "Wenxiang Chen", "Xin Guo"],
       "year": 2023,
       "arxiv_id": "2309.07864",
       "relevance": "Comprehensive survey on LLM-based agents, providing context for the multi-agent approach taken by AutoGen."
     },
     {
       "title": "A survey on large language model based autonomous agents",
       "authors": ["Lei Wang", "Chen Ma", "Xueyang Feng"],
       "year": 2023,
       "arxiv_id": "2308.11432",
       "relevance": "Survey on LLM-based autonomous agents, referenced for broader context on agent capabilities and frameworks."
     },
     {
       "title": "Self-collaboration code generation via ChatGPT",
       "authors": ["Yihong Dong", "Xue Jiang", "Zhi Jin", "Ge Li"],
       "year": 2023,
       "arxiv_id": "2304.07590",
       "relevance": "Multi-agent self-collaboration approach for code generation, relevant to understanding collaborative agent patterns for software tasks."
     },
     {
       "title": "Large language models for supply chain optimization",
       "authors": ["Beibin Li", "Konstantina Mellou", "Bo Zhang"],
       "year": 2023,
       "arxiv_id": "2307.03875",
       "relevance": "The OptiGuide system that AutoGen re-implements in Application A4, demonstrating multi-agent coding for operations research."
     },
     {
       "title": "An empirical study on challenging math problem solving with GPT-4",
       "authors": ["Yiran Wu", "Feiran Jia", "Shaokun Zhang"],
       "year": 2023,
       "arxiv_id": "2306.01337",
       "relevance": "Empirical evaluation of GPT-4 on math problem solving that provides context for AutoGen's math evaluation."
     },
     {
       "title": "Large language models as tool makers",
       "authors": ["Tianle Cai", "Xuezhi Wang", "Tengyu Ma"],
       "year": 2023,
       "arxiv_id": "2305.17126",
       "relevance": "Explores LLMs creating and using tools, relevant to the tool-backed agent capabilities in AutoGen."
     }
   ]
 }