ai-research-survey

scan.json (26283B)

---

 {
   "paper": {
     "title": "MaCTG: Multi-Agent Collaborative Thought Graph for Automatic Programming",
     "authors": [
       "Zixiao Zhao",
       "Jing Sun",
       "Zhe Hou",
       "Zhiyuan Wei",
       "Cheng-Hao Cai",
       "Miao Qiao",
       "Jin Song Dong"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2410.19245"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/MaCTG2025/MaCTG (stated in the abstract, line 70)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The BCVPP dataset is described in detail, and the GitHub repository is provided which presumably contains or links to it. Additionally, the simple projects are sourced from the publicly available 'OpenCV_Projects' repository [34]. The paper states the project can be found at the GitHub link."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware (AMD Ryzen Threadripper PRO 3995WX, 252GB RAM, two NVIDIA RTX A6000 GPUs) and Ubuntu 22.04, but does not provide a requirements.txt, Dockerfile, or detailed dependency list with library versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper itself. The GitHub repository is referenced but the paper does not contain a 'Reproducing Results' section or commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Table 1 are reported as point estimates (e.g., '83.33%') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims MaCTG outperforms baselines based on comparing accuracy numbers without any statistical significance tests (no p-values, t-tests, or similar)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., '89.09% cost reduction compared to existing multi-agent frameworks' and the ablation study shows specific accuracy drops (e.g., '14% on medium tasks and 20% on hard tasks' when removing context-aware planning). Table 1 provides full baseline-context numbers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The BCVPP dataset comprises only 90 projects (30 simple, 50 medium, 10 hard). No justification is given for why these numbers are sufficient, and the 10 hard projects in particular are very small for drawing conclusions. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be from single runs with no indication of multiple experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 includes comparisons against open-source models (CodeLLaMA-Instruct-70B, LLaMA3.3-70B, Qwen2.5-72B, LLaMA3.2-Vision-90B), proprietary tools/models (Copilot Workspace, OpenAI-o1, OpenAI-o3-mini, Claude3.7-Sonnet, DeepSeek-R1), and multi-agent frameworks (ChatDev, MetaGPT)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include very recent models: OpenAI-o3-mini-2025-01-31, Claude 3.7 Sonnet, DeepSeek-R1, LLaMA3.3-70B, Qwen2.5-72B. These are contemporary and competitive as of 2024-2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3.2 presents ablation studies removing multi-scale evaluation, context-aware planning adjustment, and replacing collaborative reasoning with independent reasoning. Results are shown in Table 1."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses only a single metric: execution accuracy (whether the output matches expected output). No other metrics such as code quality, compilation rate, or partial correctness are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated code is included. Evaluation is entirely automated via output matching. Human evaluation could assess code quality, readability, or maintainability of the generated projects."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no mention of a separate dev/test split. The entire BCVPP dataset of 90 projects appears to be the test set, with no indication that any portion was used for tuning or development."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-difficulty breakdowns (Simple, Medium, Hard) for all methods, showing performance variation across task complexity levels."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure patterns including cascading hallucinations, inconsistencies between modules (Figure 4), variable type mismatches, and incorrect file references. Section 5 discusses limitations including execution speed and stability issues."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study reports configurations that performed worse. The 'MaCTG with independent reasoning' variant scored only 44.44% overall. The paper also reports that removing context-aware planning led to 14% drop on medium and 20% drop on hard tasks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 83.33% accuracy and 89.09% cost reduction. Table 1 confirms 83.33% overall accuracy. Table 2 shows MaCTG cost of $0.61 vs. ChatDev GPT-3.5 at $5.81, supporting the 89.09% reduction claim (compared to the most expensive multi-agent framework, though the comparison base is selectively chosen)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about components improving performance (e.g., 'removing multi-scale evaluation resulted in a substantial accuracy drop'). These are supported by controlled ablation experiments where single components are removed, which is an adequate design for such claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Automatic Programming' broadly, but the evaluation is exclusively on image processing tasks using OpenCV in Python. The abstract does not bound claims to this domain. While the paper mentions the medical imaging motivation in Section 4.1, the broad title and abstract language ('auto-programming tasks', 'real-world applicability') extend well beyond what was tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for MaCTG's performance gains. For example, the performance advantage could partly stem from using DeepSeek-V3 (a very strong model) rather than the collaborative framework itself. No confound analysis is provided."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies model names like 'DeepSeek-V3', 'Qwen2.5-Coder-7B', 'GPT-4o', 'OpenAI-o1-2024-12-17', and 'OpenAI-o3-mini-2025-01-31'. However, for DeepSeek-V3, no specific version/snapshot date is given beyond the model name. 'Claude3.7-sonnet' lacks a snapshot date. 'GPT-4o' used for ChatDev and MetaGPT has no version suffix. The OpenAI o1 and o3-mini entries do include date versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes agent roles and specifications in natural language (Section 3.1.1) but does not provide the actual prompt text used for any of the agents. The prompts are described conceptually ('we prompt it to minimize the number of modules') without providing the full prompt templates or actual text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for the LLM API calls (temperature, top-p, max tokens, etc.) or for the locally deployed Qwen2.5-Coder-7B model."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in substantial detail in Section 3: the five agent types (Team Leader, Module Leader, Function Coordinator, Coder, Tester), their interactions, the graph-based reasoning protocol, context-sharing mechanism, supervised adjustment, multi-scale validation, and assembly process are all documented with diagrams (Figures 1-5)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents the dataset construction process: 30 simple projects selected from the OpenCV_Projects repository, 100 medium projects generated by OpenAI-o1 with calibration that resulted in 50 being finalized, and 10 hard projects manually identified. The filtering criteria (e.g., medium projects 'lacked coherent and logical relationships' and were reviewed/refined) are described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Discussion) contains a 'Threats to validity' subsection that discusses limitations including execution speed and stability."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity are brief and somewhat generic: (1) execution speed is slow due to locally deployed models, and (2) the dynamic agent structure introduces variability. These are more operational limitations than threats to the validity of the experimental findings. No discussion of threats like the narrow domain (only image processing), the small sample size (especially 10 hard projects), the lack of multiple runs, or potential confounds."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no discussion bounding claims to Python/OpenCV/image processing, no statement about what populations of programming tasks are excluded, and no explicit limitations on generalizability claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository (https://github.com/MaCTG2025/MaCTG) is provided. The BCVPP dataset projects and source data from the OpenCV_Projects repository are referenced, making the raw data potentially available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes how data was collected: 30 simple projects from the OpenCV_Projects GitHub repository, 100 medium projects generated by OpenAI-o1 (refined to 50 after calibration), and 10 hard projects manually identified. The sources and rationale are described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study evaluates automated programming systems on a benchmark dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents the pipeline: selection from OpenCV_Projects repository (30 simple) → generation via OpenAI-o1 (100 medium candidates) → calibration and filtering (50 medium finalized) → manual identification (10 hard). The filtering rationale (incoherent project descriptions) and final counts are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Auckland, Griffith University, Beijing Institute of Technology, Suzhou Industrial Park Monash Research Institute, National University of Singapore. None appear to be affiliated with the model providers (DeepSeek, Qwen/Alibaba)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding disclosure means this criterion cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for DeepSeek-V3 or Qwen2.5-Coder-7B. This is relevant because the BCVPP dataset includes projects from a public GitHub repository that could be in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the OpenCV_Projects repository content (used for simple projects) or similar image processing code might appear in the training data of DeepSeek-V3 or Qwen2.5-Coder. The OpenCV_Projects repo has 340+ stars and is publicly available, making contamination plausible."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The BCVPP dataset includes 30 simple projects from a public GitHub repository (OpenCV_Projects, with 340+ stars). This repository was available before the training cutoffs of the models used. No contamination analysis is provided for any of the evaluated models."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports the total API cost for evaluating on BCVPP: MaCTG costs $0.61, compared to ChatDev GPT-3.5 at $5.81, ChatDev GPT-4o at $7.83, MetaGPT GPT-4o at $5.59. The costs are directly collected from API platforms."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware configuration is specified (AMD Ryzen Threadripper PRO 3995WX, 252GB RAM, two NVIDIA RTX A6000 GPUs) and API costs are reported. However, total GPU hours or wall-clock time for the experiments are not stated. The hardware and API cost together provide a partial picture."
       }
     }
   },
   "claims": [
     {
       "claim": "MaCTG achieves state-of-the-art accuracy of 83.33% on the BCVPP auto-programming benchmark.",
       "evidence": "Table 1 shows MaCTG achieving 90.00% on simple, 78.00% on medium, and 80.00% on hard tasks, for 83.33% overall. The next best method is DeepSeek-R1 at 73.33%.",
       "supported": "moderate"
     },
     {
       "claim": "MaCTG reduces operational costs by 89.09% compared to existing multi-agent frameworks.",
       "evidence": "Table 2 shows MaCTG costs $0.61 vs. ChatDev GPT-4o at $7.83 (highest multi-agent cost). The 89.09% figure is calculated against ChatDev GPT-3.5 ($5.81), not the most expensive framework.",
       "supported": "moderate"
     },
     {
       "claim": "Context-aware planning adjustment reduces hallucinations and improves accuracy, with 14% drop on medium and 20% drop on hard tasks when removed.",
       "evidence": "Ablation in Table 1: MaCTG without context-aware planning achieves 76.67/62.00/30.00 vs. full MaCTG at 90.00/78.00/80.00. The 14% and 20% drop figures for medium and hard are confirmed (78-62=16, not 14 as claimed; 80-30=50, not 20 as claimed). The stated figures do not match the table.",
       "supported": "weak"
     },
     {
       "claim": "Collaborative reasoning outperforms independent agent reasoning, with the independent reasoning variant scoring only 44.44% overall.",
       "evidence": "Table 1 shows MaCTG with independent reasoning achieving 70.00/32.00/30.00 (44.44% overall) vs. full MaCTG at 83.33%. The difference is substantial.",
       "supported": "moderate"
     },
     {
       "claim": "MaCTG outperforms both open-source and proprietary methods across various task difficulties.",
       "evidence": "Table 1 confirms MaCTG (83.33%) outperforms all baselines including DeepSeek-R1 (73.33%), OpenAI-o1 (67.78%), and Claude 3.7 Sonnet (64.44%). However, results are from single runs on a small dataset (90 projects, only 10 hard) with no variance or significance testing.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "MaCTG, a multi-agent framework using a dynamic graph structure for collaborative reasoning, achieved 83.33% execution accuracy on the BCVPP image processing auto-programming benchmark, outperforming both individual LLMs and existing multi-agent frameworks. The hybrid model configuration (DeepSeek-V3 for planning, Qwen2.5-Coder-7B for execution) reduced API costs to $0.61 for the full benchmark evaluation. Ablation studies showed that both the context-aware planning adjustment and multi-scale validation components contribute to performance, with the collaborative reasoning approach substantially outperforming independent agent reasoning (83.33% vs 44.44%).",
   "red_flags": [
     {
       "flag": "Very small benchmark with no variance reporting",
       "detail": "The BCVPP dataset has only 90 projects (30 simple, 50 medium, 10 hard). With only 10 hard projects, the 80% accuracy for MaCTG represents 8/10 correct. Single-run results with no error bars or multiple trials make it impossible to assess result stability."
     },
     {
       "flag": "Narrow evaluation domain presented as general",
       "detail": "The evaluation is exclusively on Python image processing tasks using OpenCV, yet the title and abstract claim broad 'automatic programming' capability and 'real-world applicability'. No evidence supports generalization beyond this narrow domain."
     },
     {
       "flag": "Self-constructed benchmark evaluated only by authors",
       "detail": "The BCVPP benchmark was constructed by the authors (with 50 medium projects generated by OpenAI-o1 and curated by the authors). There is no independent validation of benchmark quality or difficulty, and the construction process could introduce biases favorable to MaCTG's design."
     },
     {
       "flag": "No contamination analysis despite using public code",
       "detail": "The simple projects come from a public GitHub repository with 340+ stars. All evaluated models may have seen this code during training. No contamination analysis is provided for any model."
     },
     {
       "flag": "Ablation study numbers inconsistent with text",
       "detail": "The paper claims removing context-aware planning adjustment reduced accuracy by '14% on medium tasks and 20% on hard tasks', but Table 1 shows drops of 16% (78-62) and 50% (80-30) respectively. This arithmetic discrepancy undermines confidence in the reported analysis."
     },
     {
       "flag": "Cost comparison uses selective baseline",
       "detail": "The 89.09% cost reduction claim compares MaCTG ($0.61) against ChatDev GPT-3.5 ($5.81), which is not the most expensive framework. Additionally, the local deployment of Qwen2.5-Coder-7B incurs GPU costs that are not included in the $0.61 figure, making the comparison asymmetric."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims are based on raw accuracy differences without any statistical testing, despite the small sample size making random variation a serious concern."
     }
   ],
   "cited_papers": [
     {
       "title": "MetaGPT: Meta Programming for Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong", "Xiawu Zheng", "Jonathan Chen"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Major multi-agent framework baseline for LLM-based software development, directly compared in experiments."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["Chen Qian", "Wei Liu", "Hongzhang Liu"],
       "year": 2024,
       "relevance": "Multi-agent code generation framework used as baseline comparison in Table 1."
     },
     {
       "title": "AgentCoder: Multi-Agent-Based Code Generation with Iterative Testing and Optimisation",
       "authors": ["Dong Huang", "Qingwen Bu", "Jie M Zhang"],
       "year": 2023,
       "arxiv_id": "2312.13010",
       "relevance": "Multi-agent code generation approach with testing agent integration, related work in multi-agent programming."
     },
     {
       "title": "Self-collaboration code generation via ChatGPT",
       "authors": ["Yihong Dong", "Xue Jiang", "Zhi Jin", "Ge Li"],
       "year": 2023,
       "arxiv_id": "2304.07590",
       "relevance": "Early multi-agent code generation approach using role-based LLM collaboration."
     },
     {
       "title": "CodeAgent: Enhancing Code Generation with Tool-Integrated Agent Systems for Real-World Repo-Level Coding Challenges",
       "authors": ["Kechi Zhang", "Jia Li", "Ge Li"],
       "year": 2024,
       "arxiv_id": "2401.07339",
       "relevance": "Agent-based code generation system integrating external tools for repository-level tasks."
     },
     {
       "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao"],
       "year": 2023,
       "arxiv_id": "2305.10601",
       "relevance": "Foundational reasoning strategy for LLMs that MaCTG's graph-based approach builds upon."
     },
     {
       "title": "Graph of Thoughts: Solving Elaborate Problems with Large Language Models",
       "authors": ["Maciej Besta", "Nils Blach", "Ales Kubicek"],
       "year": 2024,
       "relevance": "Graph-based reasoning approach for LLMs that directly inspired MaCTG's collaborative thought graph structure."
     },
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Reasoning and acting framework for LLM agents, used as inspiration for MaCTG's two-step planning process."
     },
     {
       "title": "A Survey on Large Language Model Based Autonomous Agents",
       "authors": ["Lei Wang", "Chen Ma", "Xueyang Feng"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLM-based autonomous agents covering agent frameworks and interaction strategies."
     },
     {
       "title": "Evaluating Large Language Models in Class-Level Code Generation",
       "authors": ["Xueying Du", "Mingwei Liu", "Kaixin Wang"],
       "year": 2024,
       "relevance": "Evaluation of LLMs on project-level code generation tasks, directly related to MaCTG's problem domain."
     },
     {
       "title": "A Survey on Large Language Models for Code Generation",
       "authors": ["Juyong Jiang", "Fan Wang", "Jiasi Shen"],
       "year": 2024,
       "arxiv_id": "2406.00515",
       "relevance": "Survey of LLM code generation capabilities including discussion of model selection and agent-based approaches."
     },
     {
       "title": "DeepSeek-V3 Technical Report",
       "authors": ["DeepSeek-AI"],
       "year": 2024,
       "arxiv_id": "2412.19437",
       "relevance": "Technical report for the primary proprietary model used in MaCTG's planner agents."
     }
   ]
 }