ai-research-survey

scan-v4.json (21230B)

---

 {
   "scan_version": 4,
   "paper_type": "survey",
   "paper": {
     "title": "Generative AI for Software Architecture. Applications, Trends, Challenges, and Future Directions",
     "authors": [
       "Matteo Esposito",
       "Xiaozhou Li",
       "Sergio Moreschini",
       "Noman Ahmad",
       "Tomás Cerný"
     ],
     "year": 2025,
     "venue": "Journal of Systems and Software",
     "arxiv_id": "2503.13310",
     "doi": "10.48550/arXiv.2503.13310"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about architectural decision support dominance (38%, Table 11), GPT model prevalence (62%, Table 12), few-shot prompting and RAG usage (Table 13), initial SALC stages focus (Table 14), monolithic/microservice targets (Table 15), and missing rigorous testing (93%, Table 18) are all supported by the results tables.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes descriptive claims about the state of research ('GenAI has been applied mostly to...', 'OpenAI GPT models are predominantly applied'). No causal claims about what causes or improves architectural outcomes are made.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Generative AI for Software Architecture: Applications, Challenges, and Future Directions' and conclusions like 'GenAI shows significant potential in software design' are broad, but the evidence comes from only 46 papers (36 peer-reviewed, 10 gray literature) searched through February 2025. The paper does not explicitly acknowledge that 46 papers may not represent the full landscape, especially given the rapidly evolving field.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the dominance of OpenAI GPT (62%) could reflect publication bias, marketing, or API accessibility rather than actual superiority or preference. The paper presents percentages at face value without considering confounds in what gets published.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures what is reported in published literature and frames it as how GenAI 'is utilized' in software architecture. The gap between publication patterns and actual industry practice is not explicitly discussed. The inclusion of gray literature partially addresses this but the proxy gap is not acknowledged.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated discussion structured by construct, internal, external, and conclusion validity, following Wohlin et al.'s framework.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: subjective analysis mitigated by dual-author extraction with third-author arbitration, potential non-inclusion of studies mitigated by searching eight digital libraries plus snowballing, inability to evaluate external validity of all included studies, and specific mention of applying inclusion/exclusion criteria to both title/abstract and full text.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The search is bounded temporally (after March 2022) and by topic (GenAI in software architecture), but the paper makes no explicit statements about what claims it is NOT making or what populations/settings are excluded from its conclusions.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section lists: 'Business Finland Project 6GSoft, Academy of Finland project MUFANO/349488, and National Science Foundation (NSF) Grant No. 2409933.'",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Oulu (Finland), Tampere University (Finland), University of Arizona (USA), and IIIT Hyderabad (India). None of the authors are affiliated with GenAI product companies.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are public research agencies (Business Finland, Academy of Finland, NSF) with no financial interest in any particular GenAI product or outcome.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests declaration statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "GenAI is defined in the introduction, the Software Architecture Life Cycle (SALC) phases are explicitly enumerated and operationalized in Table 14, and the MLR methodology is defined by citation to Garousi et al.; key terms are grounded in the data extraction framework (Table 6).",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The introduction explicitly enumerates five bullet-point contributions: a comprehensive synthesis, a GenAI model classification, identification of common applications and challenges, identification of research gaps, and bridging industry-research via gray literature.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 includes Table 1 comparing this work against nine prior systematic studies, explicitly stating how this MLR differs by focusing specifically on GenAI for software architecture and incorporating gray literature that prior SLRs omitted.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": true,
           "justification": "The complete search string is provided verbatim in Section 3.2.1 with all Boolean operators, wildcards, and term variants; databases are listed; a replication package with raw data is hosted on Zenodo (doi:10.5281/zenodo.15032395).",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 lists one inclusion criterion (I1) and seven exclusion criteria (E1-E7) with the screening step at which each applies (title/abstract, full text, or both), consistently applied across both white and gray literature.",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": true,
           "justification": "The paper follows the MLR guidelines of Garousi et al. [5] and Kitchenham and Charters [20], with a documented multi-stage workflow illustrated in Figure 1 covering all standard systematic review phases.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": true,
           "justification": "The full search string is displayed in a formatted box in Section 3.2.1, including all GenAI synonyms (gen AI, gen-AI, genAI, LLM, GPT*, Claude*, etc.) and software architecture variants with wildcards.",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": true,
           "justification": "White literature: ACM Digital Library, IEEEXplore, Scopus, Web of Science; gray literature: Google, Google Scholar, Bing — all explicitly named in Section 3.2.2.",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": true,
           "justification": "Table 5 provides counts at each stage for both white (621→45→27+11→36, -2 quality) and gray (433→77→5+3→10) literature, with inter-rater Cohen's k coefficients reported at each stage (k=0.71, 0.64 for white; k=0.81, 0.88 for gray).",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": true,
           "justification": "The time boundary starting March 2022 is explicitly justified by the GPT-3.5 public release; database choices are justified by Kitchenham's recommendations; the MLR approach including gray literature is justified by the rapid industry adoption outpacing academic publication.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.4 explicitly contrasts white and gray literature findings (white: formal empirical rigor, academic depth; gray: anecdotal evidence, productivity focus, less critical stance), and the introduction acknowledges 'inconsistent findings regarding the reliability of GenAI for architectural decisions.'",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": true,
           "justification": "Quality assessment is applied to white literature via an 11-question Likert checklist (Table 3, from Dyba and Dingsøyr) and to gray literature via a multi-criteria rubric (Table 4, from Garousi et al.); two white papers were excluded for failing the quality threshold.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper never discusses publication bias; it does not acknowledge that published papers about GenAI for software architecture may skew positive or that negative/null results are underrepresented in the reviewed corpus.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": true,
           "justification": "The paper systematically reports frequency counts and percentages across all findings in multiple structured tables (e.g., 62% GPT use, 38% architectural decision support, 93% lacking validation output); quantitative vote-counting synthesis is pervasive throughout the results section.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": true,
           "justification": "Section 5 derives implications from specific identified findings with citations to primary studies — e.g., validation frameworks are recommended because 93% of studies lack them; hallucination handling is recommended because five specific studies flag it as a critical issue.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "OpenAI GPT models dominate GenAI for software architecture research, used in 62% of studies",
       "evidence": "Table 12 counts 105 model references to OpenAI variants across 39 papers; confirmed in RQ1.2 summary stating 'OpenAI GPT models are the ones that rule the roost'",
       "supported": "strong"
     },
     {
       "claim": "Architectural decision support is the most frequently investigated purpose, appearing in 38% of reviewed papers",
       "evidence": "Table 11 maps 18 papers to 'Architectural Decision Support' purpose; confirmed in RQ1.1 summary",
       "supported": "strong"
     },
     {
       "claim": "93% of studies do not report any evaluation strategy for AI-generated architectural outputs",
       "evidence": "Table 18 shows 43 of 46 papers marked 'Unspecified' for validation method; only ATAM (1 paper), SAAM (1 paper), and static analysis (1 paper) are reported",
       "supported": "strong"
     },
     {
       "claim": "Requirements-to-Architecture is the most targeted SALC phase, appearing in 40% of papers",
       "evidence": "Table 14 shows 24 papers in Req-to-Arch category; stated explicitly in RQ2.1 summary",
       "supported": "strong"
     },
     {
       "claim": "Few-shot prompting is the most common technique, present in 31% of studies",
       "evidence": "Table 13 shows 16 papers using few-shot prompting; confirmed in RQ1.3 summary",
       "supported": "strong"
     },
     {
       "claim": "85% of studies involve some form of human interaction with the model",
       "evidence": "Table 13 shows 39 of 46 papers report human-model interaction; stated explicitly in RQ1.3 summary",
       "supported": "strong"
     },
     {
       "claim": "GenAI shows significant potential in software architecture but rigorous evaluation is largely absent",
       "evidence": "Supported by the 93% validation gap finding; however the positive potential claim is an editorial judgment rather than an empirically measured outcome, making it partially supported",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "qualitative",
     "meta-analysis"
   ],
   "key_findings": "This MLR of 46 papers (36 academic, 10 gray) finds that GenAI for software architecture is dominated by OpenAI GPT models (62%) and primarily targets architectural decision support (38%) and Requirements-to-Architecture transitions (40%). Despite active adoption, 93% of studies report no formal evaluation strategy for AI-generated architectural outputs, revealing a critical absence of rigorous testing. Key open challenges include LLM accuracy, hallucinations, ethics and privacy concerns, and the lack of architecture-specific datasets and standardized evaluation frameworks needed for systematic progress in the field.",
   "red_flags": [
     {
       "flag": "Small corpus, broad conclusions",
       "detail": "Only 46 papers (36 white, 10 gray) from 1054 retrieved (~4.4% inclusion rate) is a thin basis for broad conclusions about the 'state of the field' in a rapidly evolving domain."
     },
     {
       "flag": "Optimistic conclusion unsupported by evidence",
       "detail": "The conclusion states 'It is optimistic that LLM...can continue to contribute to this domain with even more astonishing outcomes' — this goes well beyond what a descriptive frequency survey can support and contradicts the paper's own finding that rigorous evaluation is largely absent."
     },
     {
       "flag": "Publication bias not discussed",
       "detail": "The paper does not acknowledge that published papers about GenAI likely skew positive, potentially inflating apparent adoption rates and success narratives in the reviewed corpus."
     },
     {
       "flag": "Gray literature heterogeneity",
       "detail": "Gray literature includes YouTube videos, blog posts, and press releases alongside more credible industry reports; mixing these sources with peer-reviewed work may distort the overall picture despite quality criteria being applied."
     },
     {
       "flag": "Boilerplate threats section",
       "detail": "The threats to validity section is largely formulaic, with internal and external validity subsections providing no specific quantified risks or concrete examples of what might be wrong."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models for software engineering: Survey and open problems (Fan et al., ICSE-FoSE 2023)",
       "relevance": "Foundational survey establishing that LLMs in SE are concentrated on code generation with limited coverage of architecture/design — directly motivates this MLR"
     },
     {
       "title": "Large language models for software engineering: A systematic literature review (Hou et al., ACM TOSEM 2024)",
       "relevance": "Prior SLR of 395 papers finding only 4 relevant to software design, quantifying the gap this MLR addresses"
     },
     {
       "title": "A survey on large language models for code generation (Jiang et al., arXiv 2024)",
       "relevance": "Related SLR of 235 papers on LLMs for code generation, contextualizing architecture-to-code as a subset"
     },
     {
       "title": "Software testing with large language models: Survey, landscape, and vision (Wang et al., IEEE TSE 2024)",
       "relevance": "Companion SLR on LLMs for testing, part of the landscape showing architecture is underserved"
     },
     {
       "title": "Artificial intelligence for software architecture: Literature review and the road ahead (Bucaioni et al., 2025)",
       "relevance": "Direct parallel work on AI for software architecture published concurrently, compared in Table 1"
     },
     {
       "title": "Software architecture meets LLMs: A systematic literature review (Schmid et al., 2025)",
       "relevance": "Very closely related SLR on LLMs for software architecture (18 papers), identified as prior work this MLR extends with broader scope and gray literature"
     },
     {
       "title": "Guidelines for including grey literature and conducting multivocal literature reviews in software engineering (Garousi et al., IST 2019)",
       "relevance": "Methodological foundation for the MLR approach, quality assessment criteria for gray literature (Table 4)"
     },
     {
       "title": "Guidelines for performing systematic literature reviews in software engineering (Kitchenham & Charters, 2007)",
       "relevance": "Core methodology guidelines for database selection, search strategy, and screening process used throughout"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Provides a taxonomy and research roadmap but no immediately usable tools or techniques for practitioners."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that 93% of studies lack GenAI output validation is noteworthy but the overall message — that GenAI in architecture is immature — largely confirms expectations."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No novel AI risk or security concerns are raised; challenges mentioned (hallucinations, accuracy) are well-known."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims; the paper is a neutral synthesis of existing literature."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, tool, or demo is provided — only a Zenodo data archive."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from University of Oulu and University of Arizona; not famous AI labs but established SE research groups. Published in Journal of Systems and Software."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "30898271",
         "title": "Visualizing quantum mechanics in an interactive simulation",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=30898271"
       },
       {
         "hn_id": "22752977",
         "title": "Agent57: Outperforming the Atari Human Benchmark",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=22752977"
       },
       {
         "hn_id": "43516923",
         "title": "UniHOPE: A Unified Approach for Hand-Only and Hand-Object Pose Estimation",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43516923"
       },
       {
         "hn_id": "43496516",
         "title": "UniHOPE: A Unified Approach for Hand-Only and Hand-Object Pose Estimation",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43496516"
       },
       {
         "hn_id": "43759600",
         "title": "In between myth and reality: AI for math – a case study in category theory",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43759600"
       },
       {
         "hn_id": "30835084",
         "title": "Visualizing quantum mechanics in an interactive simulation",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=30835084"
       },
       {
         "hn_id": "23958170",
         "title": "Agent57: Outperforming the Atari Human Benchmark",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=23958170"
       }
     ],
     "top_points": 3,
     "total_points": 13,
     "total_comments": 0
   }
 }