ai-research-survey

scan-v4.json (18907B)

---

 {
   "scan_version": 4,
   "paper_type": "survey",
   "paper": {
     "title": "Designing LLM-based Multi-Agent Systems for Software Engineering Tasks: Quality Attributes, Design Patterns and Rationale",
     "authors": [
       "Yangxiao Cai",
       "Ruiyin Li",
       "Peng Liang",
       "Mojtaba Shahin",
       "Zengyang Li"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2511.08475",
     "doi": "10.48550/arXiv.2511.08475"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about Code Generation being the most common SE task (47.9%), Functional Suitability being the most considered QA (94.7%), Role-Based Cooperation being the most used pattern (46.8%), and Improving the Quality of Generated Code being the most common rationale (44.7%) are all supported by the data in Section 4.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is descriptive, reporting what was found in the literature. It does not make causal claims about why certain patterns lead to certain outcomes.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims broad coverage of 'LLM-based Multi-Agent Systems for Software Engineering Tasks' but the study is limited to 94 papers collected before September 2024 from two surveys and arXiv only. No bounding of generalization to this specific corpus and time window is provided in the abstract or conclusions.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its observed patterns, such as whether the dominance of Code Generation reflects genuine importance or simply publication bias toward easily benchmarked tasks.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper counts papers as a proxy for design practice importance but does not discuss whether paper frequency reflects actual real-world adoption or just publication trends.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats on Validity' discusses construct validity, external validity, and reliability threats.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section discusses specific issues: individual bias in manual data extraction (mitigated by pilot extraction and multi-author review), data source selection limited to two surveys and arXiv, and methodological uncertainties resolved through author discussions.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. It does not bound claims to the specific time window (pre-September 2024), acknowledge that industrial/proprietary MAS designs are excluded, or note that its findings may not generalize beyond academic publications.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states support from NSFC Grant No. 62402348 and 62172311, and Major Science and Technology Project of Hubei Province Grant No. 2024BAA008.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: Wuhan University, RMIT University, and Central China Normal University.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (NSFC, Hubei Province) are government research agencies with no financial stake in the study outcomes.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are explicitly defined: LLM-based MASs (multiple autonomous agents collaborating via communication and responsibility specialization), Quality Attributes (per ISO/IEC 25010:2023), and Design Rationale (guiding principles behind design choices).",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three bullet-point contributions are explicitly enumerated: (1) SE task/QA identification, (2) design patterns and rationale extraction, (3) mapping relationships established among SE tasks, QAs, design patterns, and rationale.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 engages substantively with six related works across three sub-categories, explicitly comparing each to this work and articulating the gap (no prior study examined QAs + design patterns + rationale jointly for LLM-based MASs).",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": true,
           "justification": "The search is reproducible: two named seed surveys (Liu et al. 2024, Wang et al. 2025) plus arXiv SE category with the exact query '(\"large language model\" AND \"agent\")' for papers before September 30, 2024.",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": true,
           "justification": "Three explicit inclusion criteria are stated and numbered: (1) must introduce at least one MAS, (2) agents must leverage LLMs, (3) must address at least one SE task.",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": false,
           "justification": "No PRISMA flowchart or equivalent structured review protocol is used. The paper describes a research process diagram (Figure 1) but does not follow PRISMA or cite any equivalent SLR methodology standard.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": true,
           "justification": "The exact search query is provided verbatim: '(\"large language model\" AND \"agent\")' applied to arXiv's SE category.",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": true,
           "justification": "Sources are explicitly listed: Liu et al. (2024) survey, Wang et al. (2025) survey, and arXiv (Cornell University SE category). No other databases were searched.",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": false,
           "justification": "Initial counts per source are given (118 + 115 + 194 = 427, deduped to 236, final 94) but the paper does not break down how many papers were excluded by each criterion, leaving the 236→94 reduction undocumented.",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": false,
           "justification": "The time cutoff is justified only as 'when we started this study.' The decision to search only arXiv and two surveys—excluding ACM DL, IEEE Xplore, and Springer—is not justified, which is a significant scope limitation.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": false,
           "justification": "The paper is a descriptive mapping study presenting frequency counts without acknowledging any contradictions or conflicting design approaches across the 94 papers or tensions between QAs.",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": false,
           "justification": "No quality assessment of the 94 included papers is performed. All papers are treated as equal-weight sources regardless of methodological rigor, peer-review status, or empirical validity.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Publication bias is not discussed. The paper does not acknowledge that published/preprinted papers may systematically over-represent certain design patterns, QAs, or SE tasks due to reporting norms or positive framing.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": true,
           "justification": "Frequency counts and percentages are provided for all taxonomy categories and mapped across dimensions (Tables 2–6, Figures 2–4), constituting basic quantitative synthesis through vote counting, though not meta-analysis or effect size aggregation.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": true,
           "justification": "All six implications are explicitly tied to specific frequency findings (e.g., Implication 2 on Role-Based Cooperation and Maintainability directly cites Modularity being the top sub-QA and Role-Based Cooperation being the top pattern).",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Code Generation is the most common SE task addressed by LLM-based MASs (47.9%, 45/94 papers)",
       "evidence": "Figure 2 and Table 2 show 45 of 94 papers address Code Generation; next is Fault Localization at 9.6%",
       "supported": "strong"
     },
     {
       "claim": "Functional Suitability is the QA most frequently considered by designers (94.7%, 89/94 papers)",
       "evidence": "Figure 3 and Table 3 show 89 papers consider Functional Suitability, with Functional Correctness in 86 papers",
       "supported": "strong"
     },
     {
       "claim": "Role-Based Cooperation is the most frequently employed design pattern (46.8%, 44/94 papers)",
       "evidence": "Table 4 lists 44 papers using Role-Based Cooperation, followed by Self-Reflection at 36.2%",
       "supported": "strong"
     },
     {
       "claim": "Improving the Quality of Generated Code is the predominant design rationale (44.7%, 42/94 papers)",
       "evidence": "Table 5 shows 42 papers cite this rationale, followed by Simulating Human Processes at 29.8%",
       "supported": "strong"
     },
     {
       "claim": "Security is a low-priority QA among LLM-based MAS designers, considered by only 10.6% of papers",
       "evidence": "Table 3 shows only 10 papers address Security (9 Confidentiality, 1 Integrity), the same proportion as Compatibility",
       "supported": "strong"
     },
     {
       "claim": "LLM-based MASs are increasingly adopted to support the entire software lifecycle",
       "evidence": "Only 7 papers address End-to-End Software Development and 8 address End-to-End Software Maintenance (15/94 total, 16%), which is modest evidence for 'increasing' adoption",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "qualitative",
     "meta-analysis"
   ],
   "key_findings": "This systematic mapping study of 94 LLM-based MAS papers for SE tasks identifies that Code Generation dominates (47.9%), with Functional Suitability—especially Functional Correctness (91.5%)—as the primary design concern, Role-Based Cooperation as the dominant design pattern (46.8%), and improving code quality as the primary design rationale (44.7%). The paper provides taxonomic mappings across SE tasks, quality attributes (structured by ISO/IEC 25010:2023), design patterns, and rationale, with practical implications for practitioners. However, the survey does not assess the methodological quality of included papers, omits publication bias discussion, restricts search to two seed surveys plus arXiv (excluding ACM DL and IEEE Xplore), and lacks inter-rater reliability metrics for data extraction.",
   "red_flags": [
     {
       "flag": "No quality assessment of included papers",
       "detail": "All 94 papers are treated as equal-weight sources regardless of methodological rigor, peer-review status, or empirical quality. Design patterns extracted from position papers or weak empirical studies carry the same weight as patterns from rigorous studies."
     },
     {
       "flag": "Limited database coverage",
       "detail": "The search covers only arXiv's SE category plus two seed surveys, excluding ACM Digital Library, IEEE Xplore, and Springer. This likely misses published conference and journal papers not indexed in those two surveys."
     },
     {
       "flag": "Publication bias not discussed",
       "detail": "The survey does not acknowledge that published papers systematically over-report certain design choices, QAs, or rationale due to framing conventions, potentially distorting the frequency distributions reported."
     },
     {
       "flag": "Incomplete screening documentation",
       "detail": "The paper reports 236 papers before inclusion criteria and 94 after, but provides no stage-by-stage breakdown of exclusions by criterion, preventing assessment of how consistently criteria were applied."
     },
     {
       "flag": "Single-coder primary extraction",
       "detail": "The first author performed all primary data extraction; second and third authors only reviewed afterward. No independent parallel coding with inter-rater reliability metrics (e.g., Cohen's kappa) was reported."
     },
     {
       "flag": "Proxy measurement unacknowledged",
       "detail": "What designers documented in papers is treated as equivalent to what they actually prioritized, without acknowledging selective reporting: papers emphasize what differentiates them, not comprehensively documenting all design decisions."
     }
   ],
   "cited_papers": [
     {
       "title": "Large Language Model-Based Agents for Software Engineering: A Survey",
       "relevance": "Primary seed survey providing 118 initial papers and the main baseline this work builds upon and differentiates from"
     },
     {
       "title": "Agents in Software Engineering: Survey, Landscape, and Vision",
       "relevance": "Second primary seed survey providing 115 initial papers; key baseline for scope comparison"
     },
     {
       "title": "Agent Design Pattern Catalogue: A Collection of Architectural Patterns for Foundation Model based Agents",
       "relevance": "Foundational taxonomy used as starting point for design pattern classification in RQ3"
     },
     {
       "title": "LLM-Based Multi-Agent Systems for Software Engineering: Literature Review, Vision and the Road Ahead",
       "relevance": "Closely related survey by He et al. covering 71 papers; directly compared to this work to establish the research gap"
     },
     {
       "title": "MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework",
       "relevance": "Canonical example of LLM-based MAS for End-to-End Software Development; used as illustrative example throughout results"
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "relevance": "Key example of Role-Based Cooperation pattern and chat-chain task decomposition rationale"
     },
     {
       "title": "A Survey on LLM-based Multi-Agent System: Recent Advances and New Frontiers in Application",
       "relevance": "Related survey covering 125 papers from top AI venues 2023-2024; discussed in related work section"
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "relevance": "Empirical study of MAS failure modes through 200+ dialogues; provides context for reliability and fault tolerance QA findings"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Identifies 16 reusable design patterns and mapping relationships that practitioners building multi-agent SE systems can directly reference."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Findings confirm expected patterns — code generation dominates, correctness matters most, role-based cooperation is common — with no counterintuitive results."
     },
     "fear_safety": {
       "score": 0,
       "justification": "Security is mentioned as a minor quality attribute (10.6%) but no novel risks or vulnerabilities are demonstrated."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "A straightforward taxonomic survey with no controversy, no challenges to specific claims, and no conflict angle."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Dataset is publicly available on GitHub but there is no runnable tool, demo, or interactive artifact to try."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From Wuhan University and RMIT — respected but not household-name labs — and the topic is an academic taxonomy rather than a famous product."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "29250329",
         "title": "Free Will Belief as a Consequence of Model-Based Reinforcement Learning",
         "points": 2,
         "comments": 4,
         "url": "https://news.ycombinator.com/item?id=29250329",
         "created_at": "2021-11-17T08:19:50Z"
       },
       {
         "hn_id": "33617429",
         "title": "High-level synthesis for packet processing pipelines",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=33617429",
         "created_at": "2022-11-16T00:50:17Z"
       },
       {
         "hn_id": "10672783",
         "title": "Reverse Engineering Intel DRAM Addressing and Exploitation",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=10672783",
         "created_at": "2015-12-03T21:28:18Z"
       }
     ],
     "top_points": 2,
     "total_points": 6,
     "total_comments": 4
   }
 }