ai-research-survey

scan-v5.json (19059B)

---

 {
   "scan_version": 5,
   "paper_type": "survey",
   "paper": {
     "title": "Designing LLM-based Multi-Agent Systems for Software Engineering Tasks: Quality Attributes, Design Patterns and Rationale",
     "authors": [
       "Yangxiao Cai",
       "Ruiyin Li",
       "Peng Liang",
       "Mojtaba Shahin",
       "Zengyang Li"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2511.08475",
     "doi": "10.48550/arXiv.2511.08475"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All four main claims in the abstract (Code Generation 47.9%, Functional Suitability 94.7%, Role-Based Cooperation most common pattern at 46.8%, Improving Code Quality most common rationale at 44.7%) are directly supported by data tables in Sections 4.1–4.4.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The implications section makes causal-adjacent recommendations (e.g., 'Role-Based Cooperation can improve Maintainability') but the evidence is observational co-occurrence frequency from a literature mapping, which cannot support causal inference.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Implications are framed as general design guidance without explicitly bounding conclusions to the 94-paper corpus from before September 2024 or the specific limited source databases (two surveys + arXiv only).",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The dominance of code generation (47.9%) could reflect benchmark availability or publication incentives rather than actual designer priorities; the paper does not consider such alternative explanations for the observed distributions.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses frequency of mentions in papers as a proxy for what designers 'mainly focus on' without acknowledging the gap between paper-level reporting and actual practitioner design priorities.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats on Validity' explicitly addresses construct validity, external validity, and reliability threats with dedicated discussion of each.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section cites specific mitigations: pilot extraction with 5 randomly selected papers, multi-round discussions among three named authors, and named specific sources (Liu et al. and Wang et al. surveys plus arXiv SE category).",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states the date cutoff (Sept 30, 2024) and inclusion criteria but does not explicitly articulate what the results do NOT show, or bound the implications to the studied corpus and time period.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in acknowledgments: NSFC grants 62402348 and 62172311, and the Major Science and Technology Project of Hubei Province under Grant No. 2024BAA008.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Wuhan University (Cai, Li, Liang), RMIT University (Shahin), and Central China Normal University (Z. Li).",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are Chinese government science foundations (NSFC and Hubei Province), which have no direct commercial stake in the paper's taxonomic findings about LLM-based MAS design patterns.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests declaration is present; there is no statement about patents, equity, or consulting relationships anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "LLM-based MASs are defined as 'multiple autonomous agents that collaborate through communication and responsibility specialization'; Quality Attributes are categorized per ISO/IEC 25010:2023; Design Patterns are defined as 'reusable solutions that help balance quality attributes'.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions are listed in the introduction: (1) identifying SE tasks and QAs, (2) identifying design patterns and rationale, and (3) establishing mapping relationships among all four dimensions.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 provides structured review of related work across three subsections (LLM-based MASs for SE, MAS characteristics, prior surveys), and Section 2.4 explicitly articulates the research gap distinguishing this study from six named prior surveys.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": true,
           "justification": "The search query is provided ('large language model' AND 'agent' in arXiv SE category), the date cutoff (September 30, 2024) is specified, and the two prior surveys used as seed sources are named with citations.",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": true,
           "justification": "Three explicit inclusion criteria are stated: papers must introduce at least one MAS, agents must leverage LLMs, and agents must address at least one SE task; these are applied consistently.",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": false,
           "justification": "No PRISMA or other named systematic review protocol is mentioned; the paper describes a custom 4-phase process without following an established review methodology or reporting guideline.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": true,
           "justification": "The search query is explicitly provided verbatim: (\"large language model\" AND \"agent\") applied to the arXiv SE category.",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": true,
           "justification": "Sources are explicitly listed: Liu et al. 2024 survey, Wang et al. 2025 survey, and arXiv SE category; though major academic databases (IEEE Xplore, ACM DL, Scopus) were not searched.",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": true,
           "justification": "Counts are provided at each major stage: 118 (Liu et al.) + 115 (Wang et al.) + 194 (arXiv) = 427 total, 236 after deduplication, 94 after applying inclusion criteria.",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": false,
           "justification": "The paper justifies using arXiv and two surveys for comprehensiveness but does not justify excluding major academic databases (IEEE Xplore, ACM DL, Scopus), which is a significant coverage limitation left unaddressed.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": false,
           "justification": "The paper catalogues patterns across 94 papers without discussing contradictions or conflicts in findings among reviewed papers (e.g., papers advocating different patterns for the same SE task produce no tension analysis).",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": false,
           "justification": "No quality rubric, risk-of-bias assessment, or structured quality evaluation is applied to the 94 included papers; all papers are treated as equally valid sources regardless of methodological rigor.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Publication bias is not discussed; the paper does not acknowledge that arXiv preprints and conference publications skew toward positive results, or that this may inflate apparent prevalence of certain design patterns.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": true,
           "justification": "The paper provides frequency counts and percentages across all four taxonomies and cross-dimensional mapping tables (e.g., Table 6 mapping QA sub-characteristics against SE tasks with paper counts), constituting vote-counting synthesis.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": true,
           "justification": "Each of the six implications in Section 5.2 references specific frequency findings (e.g., Implication 2 on Role-Based Cooperation cites the co-occurrence with Modularity across 46 papers), tying recommendations to observed data.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Code Generation is the most common SE task addressed by LLM-based MASs at 47.9% of papers",
       "evidence": "45 of 94 papers focus on code generation, with all study references listed in Table 2",
       "supported": "strong"
     },
     {
       "claim": "Functional Suitability is the most frequently considered quality attribute, appearing in 94.7% of papers",
       "evidence": "89 of 94 papers explicitly consider Functional Suitability, with Functional Correctness the most common sub-QA (86 papers); documented in Table 3",
       "supported": "strong"
     },
     {
       "claim": "Role-Based Cooperation is the most commonly employed design pattern at 46.8% of papers",
       "evidence": "44 of 94 papers use Role-Based Cooperation, documented in Table 4 with full study citations",
       "supported": "strong"
     },
     {
       "claim": "Improving the Quality of Generated Code is the most common design rationale at 44.7% of papers",
       "evidence": "42 of 94 papers cite this rationale, listed with study references in Table 5",
       "supported": "strong"
     },
     {
       "claim": "End-to-end software lifecycle coverage remains nascent (7.4% development, 8.5% maintenance)",
       "evidence": "Only 7 papers on end-to-end development and 8 on end-to-end maintenance; attributed to lack of cross-stage benchmarks and evaluation criteria",
       "supported": "moderate"
     },
     {
       "claim": "Role-Based Cooperation improves Maintainability in LLM-based MASs",
       "evidence": "Observational co-occurrence: papers using role-based cooperation also consider modularity; no controlled comparison or causal study design",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "qualitative",
     "meta-analysis"
   ],
   "key_findings": "A mapping study of 94 papers on LLM-based MASs for SE tasks reveals heavy concentration on Code Generation (48%), with Role-Based Cooperation (47%) and Self-Reflection (36%) as the dominant design patterns and Functional Correctness (92%) as the overwhelmingly prioritized quality attribute. Improving code quality is the primary design rationale (45%), with resource optimization and efficiency also prominent. End-to-end lifecycle support remains nascent (under 9% of papers) due to absent cross-stage benchmarks and evaluation criteria, while a taxonomy of 16 design patterns, 8 QAs, and 8 rationale categories provides the first structured mapping linking SE tasks to design decisions.",
   "red_flags": [
     {
       "flag": "Limited source databases",
       "detail": "Review relies on only two prior surveys and arXiv SE category, omitting IEEE Xplore, ACM DL, and Scopus; venue-published work not captured as arXiv preprints may be systematically excluded."
     },
     {
       "flag": "No quality assessment of included papers",
       "detail": "All 94 papers are treated as equally credible sources regardless of study design or methodological rigor, conflating high-quality empirical evaluations with unvalidated system descriptions."
     },
     {
       "flag": "No PRISMA or established review protocol",
       "detail": "The review does not follow PRISMA or any named systematic review protocol, reducing reproducibility and making selection bias harder to assess."
     },
     {
       "flag": "Single-reviewer primary extraction without inter-rater reliability",
       "detail": "The first author conducted primary data extraction independently, with post-hoc review by co-authors only; no Cohen's kappa or other formal inter-rater reliability measure is reported."
     },
     {
       "flag": "Causal framing of observational findings",
       "detail": "Implications suggest patterns like Role-Based Cooperation 'improve' Maintainability, but evidence is observational co-occurrence frequency — no controlled evaluation of design pattern effects exists."
     },
     {
       "flag": "Publication bias unaddressed",
       "detail": "ArXiv self-selection and conference publication bias toward positive results are not discussed, potentially skewing the taxonomy toward approaches that appear to work rather than those commonly attempted."
     }
   ],
   "cited_papers": [
     {
       "title": "Large Language Model-Based Agents for Software Engineering: A Survey (Liu et al. 2024)",
       "relevance": "Primary seed source providing 118 of the initial paper pool; directly enables the starting corpus"
     },
     {
       "title": "Agents in Software Engineering: Survey, Landscape, and Vision (Wang et al. 2025)",
       "relevance": "Second seed source providing 115 papers; co-constitutes the initial corpus"
     },
     {
       "title": "Agent Design Pattern Catalogue: A Collection of Architectural Patterns for Foundation Model based Agents (Liu et al. 2025)",
       "relevance": "Used as classification framework for design patterns in RQ3; provides the starting taxonomy that this paper extends"
     },
     {
       "title": "MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework (Hong et al. 2023)",
       "relevance": "Prominent example system cited across multiple findings including end-to-end development, role-based cooperation, and self-reflection"
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development (Qian et al. 2024)",
       "relevance": "Key example of end-to-end software development MAS, cited for chat-chain task decomposition and role assignment"
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation (Wu et al.)",
       "relevance": "Prominent MAS framework used as example across multiple QAs, design patterns (RAG, Role-Based Cooperation), and rationale"
     },
     {
       "title": "LLM-Based Multi-Agent Systems for Software Engineering: Literature Review, Vision and the Road Ahead (He et al. 2025)",
       "relevance": "Related survey that this paper explicitly positions against to justify its distinct focus on design QAs and patterns rather than capabilities"
     },
     {
       "title": "ISO/IEC 25010:2023 Systems and software Quality Requirements and Evaluation",
       "relevance": "Foundational standard used to categorize all quality attributes in RQ2; central methodological reference throughout the paper"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The taxonomy of 16 design patterns, 8 QAs, and mapping tables provides actionable checklists for practitioners designing LLM-based MASs for specific SE tasks."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Findings confirm conventional expectations: code generation dominates, role-based cooperation and self-reflection are most common — no surprising or contrarian results are surfaced."
     },
     "fear_safety": {
       "score": 0,
       "justification": "Security is among the least-discussed QAs (10.6%) and no significant AI safety risks are surfaced by the survey findings."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Purely descriptive taxonomy study with no controversy, competing claims between research groups, or inter-community conflict."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Dataset is publicly available on GitHub (Caiyangxiao/MASDesign) allowing inspection of extraction results, though the paper itself is not a runnable artifact."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors are from Wuhan University, RMIT, and Central China Normal University — no famous lab or industry-affiliated product involved."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "29250329",
         "title": "Free Will Belief as a Consequence of Model-Based Reinforcement Learning",
         "points": 2,
         "comments": 4,
         "url": "https://news.ycombinator.com/item?id=29250329",
         "created_at": "2021-11-17T08:19:50Z"
       },
       {
         "hn_id": "33617429",
         "title": "High-level synthesis for packet processing pipelines",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=33617429",
         "created_at": "2022-11-16T00:50:17Z"
       },
       {
         "hn_id": "10672783",
         "title": "Reverse Engineering Intel DRAM Addressing and Exploitation",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=10672783",
         "created_at": "2015-12-03T21:28:18Z"
       }
     ],
     "top_points": 2,
     "total_points": 6,
     "total_comments": 4
   }
 }