ai-research-survey

scan.json (25802B)

---

 {
   "paper": {
     "title": "MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework",
     "authors": ["Sirui Hong", "Mingchen Zhuge", "Jiaqi Chen", "Xiawu Zheng", "Yuheng Cheng", "Ceyao Zhang", "Jinlin Wang", "Zili Wang", "Steven Ka Shing Yau", "Zijuan Lin", "Liyang Zhou", "Chenyu Ran", "Lingfeng Xiao", "Chenglin Wu", "Jürgen Schmidhuber"],
     "year": 2023,
     "venue": "ICLR 2024",
     "arxiv_id": "2308.00352"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided: https://github.com/geekan/MetaGPT, mentioned in the abstract."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval and MBPP are public benchmarks (YES for those), but the SoftwareDev dataset of 70 tasks is described in Table 8 with only 11 example prompts shown. The full dataset is not released with a download link."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided in the paper. Only general mentions of Python libraries (Tkinter, Pillow) in demo examples."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but the paper itself does not include commands or a reproducing results section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Figure 4 and Tables 1, 4 report point estimates only. Table 7 reports std dev for GPT variant experiments, but the main MetaGPT results have no uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims MetaGPT 'outperforms' baselines based on comparing numbers (e.g., 85.9% vs 67.0%) without any statistical significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context, e.g., '4.2% and 5.4% in Pass@1 on HumanEval and MBPP' for the feedback mechanism, and provides baseline vs. system scores throughout (e.g., GPT-4 67.0% vs MetaGPT 85.9%)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The SoftwareDev evaluation uses only 7 tasks for the main comparison (Table 1, Table 4). No justification for why 7 tasks suffice. The full dataset has 70 tasks but only 7 are used for baselines comparison."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 7 reports std dev for GPT variant experiments on HumanEval, but the main MetaGPT results (Figure 4, Tables 1, 4) show single-run numbers with no variance across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines included: AutoGPT, LangChain, AgentVerse, ChatDev for SoftwareDev; AlphaCode, Incoder, CodeGeeX, CodeGen, Codex, CodeT, PaLM, GPT-4 for HumanEval/MBPP."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4 (2023), ChatDev (2023), AgentVerse (2023), which were contemporary at submission time."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows an ablation study on roles (adding Product Manager, Architect, Project Manager incrementally). The executable feedback mechanism is also ablated (MetaGPT w/o Feedback vs MetaGPT)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Pass@1 for HumanEval/MBPP; Executability, Cost, Code Statistics, Productivity, Human Revision Cost for SoftwareDev (Table 1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Executability scores (1-4) and Human Revision Cost are evaluated by humans for the SoftwareDev benchmark (Section 4.1 Evaluation Metrics: 'human evaluations (A, E)')."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "HumanEval and MBPP are standard held-out test benchmarks. The SoftwareDev dataset is newly created by the authors."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 shows per-task executability scores for 7 individual tasks. Table 9 provides detailed per-task statistics for 11 tasks."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 9 lists specific failure modes per task (e.g., 'TypeError', 'PNG file missing', 'dependency error', 'model training method not implement'). Section D.1 discusses system limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 shows MetaGPT with Deepseek Coder 33B achieving only 1.4 executability (near failure). The ablation in Table 3 shows that removing roles degrades performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims 'generates more coherent solutions than previous chat-based multi-agent systems' is supported by Tables 1 and 4. State-of-the-art on HumanEval/MBPP is supported by Figure 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'SOPs improve collaboration' are supported by controlled ablations (Table 3 — adding roles incrementally) and the feedback mechanism ablation. Single-variable manipulation in ablation design is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Multi-Agent Collaborative Framework' broadly but evaluation is only on Python code generation tasks (HumanEval, MBPP, SoftwareDev). The abstract says 'collaborative software engineering benchmarks' but scope is limited to GPT-4 backend, Python, and specific task types."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for results. The improvement could partly be due to GPT-4's inherent capability rather than the SOP framework, or to the additional token budget used. These confounds are not addressed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures Pass@1 and executability on toy tasks but frames results as evidence for 'collaborative software engineering' capability. No discussion of whether these benchmarks capture real software engineering complexity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-4' and 'GPT-3.5-Turbo' without version snapshots in the main experiments. Table 7 mentions 'gpt-4-0613' and 'gpt-3.5-turbo-0613' but only for the supplementary HumanEval variant experiment, not for the main MetaGPT results."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompts are described conceptually (role definitions, SOP structure) but the actual prompt text sent to GPT-4 for each role is not provided. Appendix B shows demo outputs but not the prompts that generated them."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other API parameters are reported for the main experiments."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail: role definitions (Section 3.1), communication protocol with publish-subscribe (Section 3.2), executable feedback mechanism (Section 3.3), with workflow diagrams (Figures 1-3)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "For HumanEval/MBPP: 'we slightly modified the prompts to align with response format requirements' but the modifications are not described. For SoftwareDev: task selection criteria for the 7 comparison tasks are described as 'randomly select seven representative tasks' without documenting the selection."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section D.1 'Limitation' discusses system-side and human-user-side limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations in D.1 are about capability gaps ('cannot fully cater to specific scenarios, such as UI and front-end') rather than threats to the validity of the evaluation. No discussion of whether the results could be invalid or misleading."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do NOT show. The paper does not bound its claims to specific settings, languages, or task types."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental outputs (generated code, logs, intermediate artifacts) are released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "HumanEval and MBPP are well-described standard benchmarks. The SoftwareDev dataset construction is described in Section 4.1 and Table 8 with task descriptions and scope (Figure 5)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited. Human evaluators are mentioned for executability scoring but their selection/identity is not described — however this is a benchmark eval, not a human subjects study. NA since data sources are standard benchmarks and a custom dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw model outputs to final scores is not documented. How executability was scored, who scored it, and inter-rater reliability are not described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section or grant acknowledgments found. The acknowledgment section thanks individuals for editing help but does not disclose funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: DeepWisdom, KAUST, Xiamen University, CUHK-Shenzhen, etc. Chenglin Wu is identified as CEO of DeepWisdom."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "DeepWisdom is a company that initiated and hosts MetaGPT. The CEO is an author. The paper evaluates their own product. This is a clear non-independent funding/ownership relationship, not disclosed as a conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. Chenglin Wu is CEO of DeepWisdom which hosts MetaGPT and later launched AgentStore (Appendix A.2), but no financial interest declaration is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4 on HumanEval and MBPP but does not state GPT-4's training data cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021 and MBPP in 2021. GPT-4 could have seen these benchmarks during training. No discussion of this overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval and MBPP have been publicly available since 2021, well before GPT-4's training. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study. Human evaluators scored executability but this is not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study conducted."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports token usage (24,613 / 31,255) and running time (503s / 541s). Table 3 reports expense per run (e.g., $0.915 to $1.385). Table 9 reports money costs per task."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1, 3, and 9 report running times, token usage, and dollar costs. Table 5 reports time per run for different backends."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main MetaGPT results are single-run. Table 7 reports 5 runs with std dev for GPT variant experiments, but the core MetaGPT vs. baseline comparisons have no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of runs for the main MetaGPT experiments is not stated. Table 7 states '5 times' for the GPT variant experiment only."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. The maximum 3 retries for executable feedback appears tuned but no search budget is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why the specific configuration (roles, retry limit, prompt design) was chosen. The ablation shows incremental roles but doesn't explain how the final 4-role + feedback configuration was selected."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors compare their system (MetaGPT) against their own re-implementations/runs of baselines (AutoGPT, LangChain, AgentVerse, ChatDev) without acknowledging self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "MetaGPT uses significantly more tokens than ChatDev (31,255 vs 19,292) but this compute difference is not discussed as a confound for the performance comparison."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval/MBPP (single-function problems) actually measure multi-agent software engineering capability. The SoftwareDev benchmark validity is not questioned."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MetaGPT uses a sophisticated multi-agent scaffold while baselines use simpler scaffolds. The paper attributes improvement to the SOP design but does not control for the overall scaffold complexity/token budget difference."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) predate GPT-4's training. No discussion of temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks answer information. The multi-pass design (PRD → design → code → test) could interact with model memorization, but this is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval/MBPP problems share structure with GPT-4's training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "MetaGPT achieves 85.9% and 87.7% Pass@1 on HumanEval and MBPP respectively, state-of-the-art performance.",
       "evidence": "Figure 4 shows Pass@1 comparisons. Prior best was Codex+CodeT at 65.8%/67.7% and GPT-4 at 67.0%.",
       "supported": "moderate"
     },
     {
       "claim": "MetaGPT achieves 100% task completion rate on the SoftwareDev benchmark.",
       "evidence": "Mentioned in Section 1 but Table 4 shows scores ranging from 3 to 4 (not all perfect), with Flappy Bird at 3. The 100% completion claim appears to mean 'produces output' not 'works perfectly'.",
       "supported": "weak"
     },
     {
       "claim": "The executable feedback mechanism improves Pass@1 by 4.2% on HumanEval and 5.4% on MBPP.",
       "evidence": "Figure 4 compares MetaGPT w/o Feedback (81.7%, 82.3%) vs MetaGPT (85.9%, 87.7%). Table 1 shows executability improvement from 3.67 to 3.75.",
       "supported": "moderate"
     },
     {
       "claim": "MetaGPT outperforms ChatDev on the SoftwareDev dataset in nearly all metrics.",
       "evidence": "Table 1 shows MetaGPT scores 3.75 vs ChatDev 2.25 executability, 0.83 vs 2.5 human revision cost, 124.3 vs 248.9 productivity ratio.",
       "supported": "moderate"
     },
     {
       "claim": "SOPs significantly improve multi-agent collaboration and reduce hallucination cascading.",
       "evidence": "Table 3 ablation shows adding roles improves executability (1.0 → 4.0) and reduces revisions (10 → 2.5). Qualitative argument about structured communication but no direct hallucination measurement.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MetaGPT introduces a multi-agent framework encoding software development SOPs into LLM prompt sequences, achieving 85.9% and 87.7% Pass@1 on HumanEval and MBPP. On a custom SoftwareDev benchmark of 70 tasks, MetaGPT scores 3.75/4.0 executability vs ChatDev's 2.25, while requiring fewer human revisions (0.83 vs 2.5). Ablations show that adding specialized roles and executable feedback both contribute to improved performance, though the system uses significantly more tokens than baselines.",
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "DeepWisdom CEO (Chenglin Wu) is a corresponding author. DeepWisdom initiated MetaGPT and later monetized it through AgentStore. The paper evaluates their own framework with no independent evaluation or conflict disclosure."
     },
     {
       "flag": "Tiny evaluation sample for main claims",
       "detail": "The SoftwareDev comparison uses only 7 tasks despite 70 being available. Table 4 shows per-task scores but N=7 is very small for the claims made. The full 70-task results (Table 9) are only shown for MetaGPT w/o feedback, not for baselines."
     },
     {
       "flag": "No variance or multiple runs for main results",
       "detail": "Core HumanEval/MBPP results and SoftwareDev comparisons appear to be single runs. LLM outputs are stochastic; single-run comparisons are unreliable."
     },
     {
       "flag": "Benchmark contamination risk",
       "detail": "HumanEval and MBPP were published in 2021. GPT-4 was likely trained on data including these benchmarks. The multi-agent pipeline may amplify memorized solutions through structured prompting, inflating scores beyond what the framework itself contributes."
     },
     {
       "flag": "Unfair compute comparison",
       "detail": "MetaGPT uses 31,255 tokens vs ChatDev's 19,292 (~62% more). The additional tokens buy more opportunities for correction. Performance improvement may partly reflect token budget rather than architectural innovation."
     },
     {
       "flag": "100% completion rate claim is misleading",
       "detail": "The abstract and Section 1 claim '100% task completion rate' but Table 4 shows Flappy Bird scores only 3/4 (not flawless). The claim appears to mean 'produces some output' which is a very low bar."
     }
   ],
   "cited_papers": [
     {
       "title": "Communicative Agents for Software Development",
       "authors": ["Chen Qian", "Xin Cong", "Cheng Yang", "Weize Chen"],
       "year": 2023,
       "relevance": "ChatDev: main baseline, multi-agent framework for code generation using chat-based collaboration."
     },
     {
       "title": "CAMEL: Communicative Agents for 'Mind' Exploration of Large Scale Language Model Society",
       "authors": ["Guohao Li"],
       "year": 2023,
       "relevance": "Role-playing multi-agent framework; closest prior work on multi-agent collaboration for programming."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "relevance": "Introduces HumanEval benchmark and Codex; primary evaluation benchmark used in this paper."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena"],
       "year": 2021,
       "relevance": "Introduces MBPP benchmark; second primary evaluation benchmark used."
     },
     {
       "title": "Generative Agents: Interactive Simulacra of Human Behavior",
       "authors": ["Joon Sung Park"],
       "year": 2023,
       "relevance": "Foundational multi-agent simulation work; MetaGPT draws analogy to simulated towns."
     },
     {
       "title": "Reflexion: An Autonomous Agent with Dynamic Memory and Self-Reflection",
       "authors": ["Noah Shinn", "Beck Labash", "Ashwin Gopinath"],
       "year": 2023,
       "relevance": "Self-reflection mechanism for LLM agents; related to MetaGPT's executable feedback."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao"],
       "year": 2022,
       "relevance": "ReAct-style reasoning loop used as the base agent behavior pattern in MetaGPT."
     },
     {
       "title": "Voyager: An Open-Ended Embodied Agent with Large Language Models",
       "authors": ["Guanzhi Wang"],
       "year": 2023,
       "relevance": "Open-ended LLM agent with self-improvement; MetaGPT draws parallel to Minecraft sandbox."
     },
     {
       "title": "AgentVerse: Facilitating Multi-Agent Collaboration and Exploring Emergent Behaviors in Agents",
       "authors": ["Weize Chen"],
       "year": 2023,
       "relevance": "Multi-agent collaboration framework; direct baseline comparison in SoftwareDev evaluation."
     },
     {
       "title": "Self-collaboration Code Generation via ChatGPT",
       "authors": ["Yihong Dong"],
       "year": 2023,
       "relevance": "Multi-persona self-collaboration for code generation; related multi-agent approach."
     },
     {
       "title": "Competition-Level Code Generation with AlphaCode",
       "authors": ["Yujia Li"],
       "year": 2022,
       "relevance": "Competition-level code generation baseline compared in HumanEval results."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei"],
       "year": 2022,
       "relevance": "CoT prompting foundation; MetaGPT's SOP design compared to injecting CoT in LLMs."
     }
   ]
 }