ai-research-survey

scan.json (22266B)

---

 {
   "paper": {
     "title": "Towards Engineering Multi-Agent LLMs: A Protocol-Driven Approach",
     "authors": ["Zhenyu Mao", "Jacky Keung", "Fengji Zhang", "Shuo Liu", "Yifei Wang", "Jialong Li"],
     "year": 2025,
     "venue": "Asia-Pacific Software Engineering Conference",
     "arxiv_id": "2510.12120",
     "doi": "10.1109/APSEC66846.2025.00100"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SEMAP, a protocol-layer methodology applying SE principles (behavioral contracts, structured messaging, lifecycle-guided execution) to multi-agent LLMs, reduces failures by up to 69.6% in function-level development and 56.7% in deployment-level development compared to a MetaGPT baseline. In vulnerability detection, failure reductions of up to 47.4% (Python) and 28.2% (C/C++) are observed. SEMAP shows more stable failure reduction across collaboration rounds than the baseline.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code release mentioned. The conclusion mentions 'releasing artifacts for reproducibility' as future work."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: HumanEval, ProgramDev, Devign (devign100 subset), and CVEFixes (vudenc100 subset). However, the specific 100-sample subsets (devign100, vudenc100) are constructed by the authors and not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or hardware details provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions provided. Artifacts release is listed as future work."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables I and II report only point estimates (raw counts and percentage changes) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims of failure reduction (e.g., '69.6% reduction') are made by comparing raw counts without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage reductions are reported with baseline context (e.g., 'from 256 to 92' = 64.1%), providing enough information to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 100-sample subsets were chosen for vulnerability detection, or why HumanEval's 164 problems are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures reported. Results appear to be from single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "MetaGPT framework is used as the baseline system for both development and vulnerability detection tasks."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "Only one baseline (MetaGPT) is used. The conclusion acknowledges the need to 'compare against more baselines, including single-agent LLMs and domain-specific detectors' as future work."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "SEMAP has three components (contracts, messaging, lifecycle verification) but no ablation study isolates their individual contributions. The conclusion lists 'ablation studies' as future work."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Failures are categorized into three types (under-specification, inter-agent misalignment, task verification) and reported separately, plus total counts and round-by-round trends."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Failure categorization is done entirely by LLM-as-a-Judge (gpt-4o-2024-08-06), not human evaluation. No human review of outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmark test sets (HumanEval, ProgramDev, devign100, vudenc100) are used for evaluation, not for tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables I and II provide per-failure-category breakdowns across all tasks and models."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative failure analysis or specific failure examples are discussed. Results are purely quantitative."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Some weak results are reported: e.g., ProgramDev with GPT-4.1-nano shows only 12.6% total reduction, and 0.0% reduction in inter-agent misalignment; devign100 with DeepSeek shows only 8.3% reduction."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims ('up to 69.6% reduction', '56.7%', '47.4%', '28.2%') are supported by Tables I and II."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SEMAP 'reduces failures' and 'mitigates' issues (causal language), but the comparison is only against one baseline (MetaGPT) without controlling for confounds like the A2A infrastructure itself or the additional prompt engineering in SEMAP."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims a general 'protocol-driven approach' for 'multi-agent LLMs' but results are only from two models (DeepSeek-V3-0324 and gpt-4.1-nano) on four specific datasets. The paper does not bound its generalizations to these settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. The improvements could be due to more detailed prompting, the A2A infrastructure, or simply more structured agent instructions rather than the three SE principles claimed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures failure counts (as judged by an LLM) and frames this as 'system robustness' and 'effectiveness' without discussing the gap between LLM-judged failures and actual system quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model versions are specified: 'DeepSeek-V3-0324' and 'gpt-4.1-nano-2025-04-14', plus 'gpt-4o-2024-08-06' for the judge."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided. The paper describes the roles and contracts conceptually but does not provide the actual prompts or system instructions given to agents."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for any of the models used."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The SEMAP protocol is described in detail in Section III, including behavioral contracts, structured messaging format, and lifecycle FSM. The architecture (centralized 5-agent for dev, decentralized 3-agent for vuln detection) is specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Dataset construction is documented: devign100 is '100-sample subset...constructed by randomly selecting 50 vulnerable and 50 safe C/C++ functions' from Devign; vudenc100 similarly from CVEFixes with labeling criteria described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section. The conclusion mentions future work items that implicitly acknowledge limitations, but there is no substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity discussed. The paper does not address concerns like LLM-as-a-Judge reliability, single-baseline comparison, or single-run results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion mentions scaling to 'larger datasets, agent populations, and longer workflows' as future work, implicitly acknowledging current scope, but does not explicitly state what the results do NOT show."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (agent outputs, failure logs, LLM judge outputs) is made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Dataset construction procedure is described: random sampling from Devign and CVEFixes with balanced classes (50/50), labeling criteria for vulnerability defined."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All evaluation is automated using benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw benchmark problems to failure counts is not fully documented. How the LLM-as-a-Judge categorizes failures, what its prompt looks like, and how ambiguous cases are handled is not described beyond citing [19]."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations (City University of Hong Kong, Waseda University) are clearly stated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement present."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for DeepSeek-V3-0324 or gpt-4.1-nano. HumanEval (2021) is likely in both models' training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval or other benchmark problems appeared in the models' training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021 and is widely known to be contaminated in modern LLMs. This is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API costs, or latency reported. The conclusion mentions 'measuring resource overhead' as future work."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget stated (API spend, GPU hours, wall-clock time)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple seeds or seed sensitivity. Results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Number of runs per experiment is not stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported. The number of collaboration rounds (5) is stated but no tuning process described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the SEMAP configuration was selected or whether alternatives were tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons across 4 datasets × 2 models × 3 failure categories with no statistical tests at all, let alone corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement SEMAP and compare against MetaGPT baseline without acknowledging author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "SEMAP adds protocol overhead (contracts, structured messaging, lifecycle FSM) compared to MetaGPT baseline but compute cost differences are not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval function-level tasks and 100-sample subsets are valid measures of multi-agent system quality."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SEMAP uses A2A infrastructure while baseline uses MetaGPT — different scaffolds. Performance differences could be due to the scaffold rather than the SE principles. This confound is not addressed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) predates both models' training. ProgramDev and devign100/vudenc100 temporal relationships to model training not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides information not available in realistic usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods applied."
       }
     }
   },
   "claims": [
     {
       "claim": "SEMAP reduces total failures by up to 69.6% for function-level development (HumanEval with DeepSeek)",
       "evidence": "Table I: DeepSeek baseline 112 total failures → SEMAP 34 (69.6% reduction)",
       "supported": "moderate"
     },
     {
       "claim": "SEMAP reduces total failures by up to 56.7% for deployment-level development (ProgramDev with DeepSeek)",
       "evidence": "Table I: DeepSeek baseline 67 → SEMAP 29 (56.7% reduction)",
       "supported": "moderate"
     },
     {
       "claim": "SEMAP reduces vulnerability detection failures by up to 47.4% (Python with GPT-4.1-nano)",
       "evidence": "Table II: GPT-4.1-nano baseline 38 → SEMAP 20 (47.4% reduction on vudenc100)",
       "supported": "weak"
     },
     {
       "claim": "SEMAP promotes more stable failure reduction across collaboration rounds",
       "evidence": "Figure 2 shows SEMAP trends declining more steadily than baseline across 5 rounds",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "LLM-as-a-Judge evaluation",
       "detail": "All failure categorization is done by gpt-4o-2024-08-06 using the pipeline from [19]. The reliability of this automated judge is not validated in this paper, and no human evaluation corroborates the LLM's assessments."
     },
     {
       "flag": "Single baseline comparison",
       "detail": "Only MetaGPT is used as a baseline. No comparison against single-agent LLMs, other multi-agent frameworks (AutoGen, ChatDev), or ablated SEMAP variants."
     },
     {
       "flag": "No statistical rigor",
       "detail": "All claims of improvement are based on raw count comparisons from what appear to be single runs. No significance tests, confidence intervals, or variance reported."
     },
     {
       "flag": "Scaffold confound",
       "detail": "SEMAP is built on Google A2A while baseline uses MetaGPT. Observed improvements may be due to infrastructure differences rather than the proposed SE principles."
     },
     {
       "flag": "Cherry-picked 'up to' framing",
       "detail": "Abstract highlights best-case reductions ('up to 69.6%') while some configurations show much smaller improvements (8.3% for devign100 with DeepSeek, 12.6% for ProgramDev with GPT-4.1-nano)."
     },
     {
       "flag": "Benchmark contamination risk",
       "detail": "HumanEval (2021) is likely heavily contaminated in both DeepSeek-V3 and GPT-4.1 training data, potentially confounding the development task results."
     },
     {
       "flag": "No ablation study",
       "detail": "SEMAP combines three principles but no ablation isolates individual contributions. Impossible to know which components drive improvements."
     }
   ],
   "cited_papers": [
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["M. Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Introduces MAST failure taxonomy used as the evaluation framework and provides the ProgramDev benchmark and LLM-as-a-Judge pipeline."
     },
     {
       "title": "A survey of AI agent protocols",
       "authors": ["Y. Yang"],
       "year": 2025,
       "arxiv_id": "2504.16736",
       "relevance": "Survey categorizing multi-agent communication protocols along context/inter-agent and general/domain-specific dimensions."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark used for function-level code generation evaluation."
     },
     {
       "title": "LLM-based multi-agent systems for software engineering: Literature review, vision and the road ahead",
       "authors": ["J. He"],
       "year": 2024,
       "relevance": "Literature review of multi-agent LLMs for SE tasks, directly relevant to the survey scope."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": ["Q. Qian"],
       "year": 2023,
       "relevance": "Multi-agent framework for collaborative software development using LLM agents."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversation",
       "authors": ["Q. Wu"],
       "year": 2023,
       "relevance": "Major multi-agent LLM framework enabling conversational agent workflows."
     },
     {
       "title": "Devign: Effective vulnerability identification by learning comprehensive program semantics via graph neural networks",
       "authors": ["Y. Zhou"],
       "year": 2019,
       "relevance": "Source dataset for vulnerability detection evaluation in multi-agent systems."
     },
     {
       "title": "MAGIS: LLM-based multi-agent framework for GitHub issue resolution",
       "authors": ["W. Tao"],
       "year": 2024,
       "arxiv_id": "2403.17927",
       "relevance": "Multi-agent framework for software maintenance tasks (issue resolution)."
     },
     {
       "title": "Multi-role consensus through LLMs discussions for vulnerability detection",
       "authors": ["Z. Mao"],
       "year": 2024,
       "relevance": "Prior work by same authors on multi-agent LLM vulnerability detection."
     }
   ]
 }