ai-research-survey

scan.json (27558B)

---

 {
   "paper": {
     "title": "Generative AI for Software Architecture. Applications, Challenges, and Future Directions",
     "authors": [
       "Matteo Esposito",
       "Xiaozhou Li",
       "Sergio Moreschini",
       "Noman Ahmad",
       "Tomas Cerny",
       "Karthik Vaidhyanathan",
       "Valentina Lenarduzzi",
       "Davide Taibi"
     ],
     "year": 2025,
     "venue": "Journal of Systems and Software",
     "arxiv_id": "2503.13310",
     "doi": "10.48550/arXiv.2503.13310"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a replication package on Zenodo (doi:10.5281/zenodo.15032395) described as containing 'raw data, and the MLR workflow,' but no source code or analysis scripts are explicitly mentioned. The replication package appears to be data/documentation rather than executable code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states 'We provide our raw data, and the MLR workflow in our replication package hosted on Zenodo' with a DOI link (10.5281/zenodo.15032395)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or tool version specifications are provided. For an MLR the relevant 'environment' would be the tools used for search, screening, and analysis — none are specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a replication package is provided on Zenodo, the paper does not include step-by-step reproduction instructions. The methodology section describes the process conceptually but not as a reproducible script or protocol with exact commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that reports descriptive counts and percentages of papers. No experimental results requiring confidence intervals are produced."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a systematic literature review reporting descriptive statistics (counts, percentages) of its included studies. No comparative statistical claims requiring significance tests are made."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental comparisons are performed. The paper reports descriptive frequencies of themes found across reviewed papers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The final sample of 46 papers (36 white, 10 gray) is not justified as sufficient for the breadth of claims made. No discussion of whether this sample adequately represents the field or whether saturation was reached."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs or repeated measurements are performed. The paper reports counts from a single systematic search process."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides a detailed comparison of this MLR against 10 prior systematic studies (Fan et al., Hou et al., Jiang et al., Wang et al., Marques et al., Santos et al., Saucedo & Rodriguez, Bucaioni et al., Schmid et al.), situating this work relative to existing reviews."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The compared reviews in Table 1 include recent works: Bucaioni et al. (2025), Schmid et al. (2025), Santos et al. (2024), and others from 2023-2024. These represent the current state of related review literature."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate a system using metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of a system's outputs is not applicable to a literature review."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experimental evaluation requiring train/test splits is performed."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down extensively by category: purpose of LLM usage (Table 11), models used (Table 12), how GenAI is used (Table 13), SALC phases (Table 14), architectural styles (Table 15), modeling languages (Table 16), validation methods (Table 18), use cases (Table 19), future challenges (Table 21), and white vs. gray literature (Table 7)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses significant gaps found in the reviewed literature: 93% of studies lack validation methods (Table 18), 68% omit architectural style details (Table 15), 74% lack formal modeling (Table 16), and 15% of studies don't mention future challenges (Section 4.4)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: rigorous testing of GenAI outputs is 'typically missing,' 93% of studies provide no validation, Architecture-to-Architecture transitions are nearly unexplored (3%), and there is a 'paucity of examples addressing system-level reasoning, trade-off analysis, or performance modeling.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about architectural decision support dominance (38%, Table 11), GPT model prevalence (62%, Table 12), few-shot prompting and RAG usage (Table 13), initial SALC stages focus (Table 14), monolithic/microservice targets (Table 15), and missing rigorous testing (93%, Table 18) are all supported by the results tables."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes descriptive claims about the state of research ('GenAI has been applied mostly to...', 'OpenAI GPT models are predominantly applied'). No causal claims about what causes or improves architectural outcomes are made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Generative AI for Software Architecture: Applications, Challenges, and Future Directions' and conclusions like 'GenAI shows significant potential in software design' are broad, but the evidence comes from only 46 papers (36 peer-reviewed, 10 gray literature) searched through February 2025. The paper does not explicitly acknowledge that 46 papers may not represent the full landscape, especially given the rapidly evolving field."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the dominance of OpenAI GPT (62%) could reflect publication bias, marketing, or API accessibility rather than actual superiority or preference. The paper presents percentages at face value without considering confounds in what gets published."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures what is reported in published literature and frames it as how GenAI 'is utilized' in software architecture. The gap between publication patterns and actual industry practice is not explicitly discussed. The inclusion of gray literature partially addresses this but the proxy gap is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use any AI models for its own analysis. It is a literature review. (The authors acknowledge using ChatGPT for language editing but this is not part of the methodology.)"
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting as part of its methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No models are run as part of the study methodology."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this literature review."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper selection pipeline is documented in detail: search terms (Section 3.2.1), databases (Section 3.2.2), inclusion/exclusion criteria with specific rules (Table 2), step-by-step counts at each filtering stage (Table 5: 1054 → title/abstract screening → full reading → snowballing → quality assessment → 46 final papers), and inter-rater agreement (Cohen's kappa reported at each stage)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated discussion structured by construct, internal, external, and conclusion validity, following Wohlin et al.'s framework."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: subjective analysis mitigated by dual-author extraction with third-author arbitration, potential non-inclusion of studies mitigated by searching eight digital libraries plus snowballing, inability to evaluate external validity of all included studies, and specific mention of applying inclusion/exclusion criteria to both title/abstract and full text."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The search is bounded temporally (after March 2022) and by topic (GenAI in software architecture), but the paper makes no explicit statements about what claims it is NOT making or what populations/settings are excluded from its conclusions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We provide our raw data, and the MLR workflow in our replication package hosted on Zenodo' (doi:10.5281/zenodo.15032395), enabling independent verification of the extracted data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail in Section 3: search terms (Section 3.2.1), four white-literature databases and three gray-literature search engines (Section 3.2.2), inclusion/exclusion criteria (Table 2), and the open coding approach for data extraction (Section 3.3, Table 6)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The paper sources are academic databases and search engines, which are standard and described in the search strategy."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 documents the full pipeline: 621 white papers → title/abstract screening (-576) → full reading (-18) → snowballing (+11) → quality assessment (-2) → 36 final. Gray: 433 → -356 → -70 → +3 → 10 final. Cohen's kappa is reported at each screening stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section lists: 'Business Finland Project 6GSoft, Academy of Finland project MUFANO/349488, and National Science Foundation (NSF) Grant No. 2409933.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Oulu (Finland), Tampere University (Finland), University of Arizona (USA), and IIIT Hyderabad (India). None of the authors are affiliated with GenAI product companies."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are public research agencies (Business Finland, Academy of Finland, NSF) with no financial interest in any particular GenAI product or outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests declaration statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a literature review that does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a literature review that does not evaluate any pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a literature review that does not evaluate any pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this literature review."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this literature review."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this literature review."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. (The paper does have inclusion/exclusion criteria for papers, which is covered under data_preprocessing_documented.)"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no inference or computational method of its own."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The paper follows the MLR protocol of Garousi et al. [5] and Kitchenham's SLR guidelines [20]. It includes a systematic search strategy (Figure 1), defined search terms (Section 3.2.1), four white-literature databases, three gray-literature search engines, structured inclusion/exclusion criteria (Table 2), inter-rater reliability measures (Cohen's kappa), and snowballing following Wohlin's guidelines [22]."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": true,
         "justification": "White literature is assessed using 11 quality assessment criteria (Table 3) on a 5-point Likert scale, with a minimum score of 2 required for inclusion. Gray literature uses a separate quality rubric (Table 4) covering authority, methodology, objectivity, date, novelty, impact, and outlet control, with a minimum average score of 0.5. Two papers were excluded via quality assessment."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Publication bias is not explicitly discussed. While the inclusion of gray literature partially addresses the gap between academic and industry perspectives, the paper does not discuss whether positive results are overrepresented in the reviewed studies, use funnel plots, or acknowledge systematic bias in what gets published about GenAI."
       }
     }
   },
   "scan_version": 3,
   "active_modules": [
     "survey_methodology"
   ],
   "claims": [
     {
       "claim": "Architectural decision support is the most frequent purpose for GenAI in software architecture (38% of studies).",
       "evidence": "Table 11 (RQ1.1) lists 18 studies under 'Architectural Decision Support' out of 47 total coded purposes.",
       "supported": "strong"
     },
     {
       "claim": "OpenAI GPT models dominate the research landscape at 62% of model usage.",
       "evidence": "Table 12 (RQ1.2) counts 105 GPT model instances across studies, representing 62% of all model usage.",
       "supported": "strong"
     },
     {
       "claim": "Few-shot prompting is the most common prompt engineering technique (31%).",
       "evidence": "Table 13 (RQ1.3) reports 16 studies using few-shot prompting out of 52 coded instances.",
       "supported": "strong"
     },
     {
       "claim": "93% of studies do not report any validation method for GenAI-generated architectural outputs.",
       "evidence": "Table 18 (RQ2.4) shows 43 of 46 studies listed as 'Unspecified' for architecture analysis method, with only ATAM, SAAM, and static analysis reported (1 study each).",
       "supported": "strong"
     },
     {
       "claim": "Requirement-to-Architecture is the most targeted SALC phase (40%) and Architecture-to-Code is second (32%).",
       "evidence": "Table 14 (RQ2.1) reports 24 studies targeting Req-to-Arch and 19 targeting Arch-to-Code.",
       "supported": "strong"
     },
     {
       "claim": "85% of studies involve some form of human interaction with the GenAI model.",
       "evidence": "Table 13 (RQ1.3) reports 39 of 46 studies involving human-model interaction.",
       "supported": "strong"
     },
     {
       "claim": "LLM accuracy (15%) and hallucinations (8%) are the most cited future challenges.",
       "evidence": "Table 21 (RQ3) lists 9 studies citing LLM accuracy and 5 citing hallucinations as future challenges.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "meta-analysis"
   ],
   "key_findings": "This MLR of 46 studies (36 peer-reviewed, 10 gray literature) finds that GenAI in software architecture is primarily used for architectural decision support (38%) and reverse engineering (19%), with OpenAI GPT models dominating at 62% of usage. GenAI is applied mostly in early SALC phases (Requirements-to-Architecture 40%, Architecture-to-Code 32%), while Architecture-to-Architecture transformation is nearly unexplored (3%). A critical finding is that 93% of studies lack any formal validation method for GenAI-generated architectural outputs, and most studies (85%) still require human interaction, indicating GenAI serves as an assistive tool rather than an autonomous decision-maker.",
   "red_flags": [
     {
       "flag": "Small sample for broad claims",
       "detail": "Only 46 papers (36 white, 10 gray) support claims about the entire landscape of 'Generative AI for Software Architecture.' The title and conclusions suggest comprehensive coverage but the evidence base is thin for such sweeping claims, especially given the rapidly evolving field."
     },
     {
       "flag": "Gray literature quality concerns",
       "detail": "10 of 46 included sources (22%) are gray literature including blog posts (4), a YouTube video (1), and theses (2). While the MLR methodology accounts for this, gray literature sources like Medium blog posts and YouTube videos may introduce anecdotal or biased perspectives that are weighted equally with peer-reviewed work in the frequency counts."
     },
     {
       "flag": "No quality-weighted synthesis",
       "detail": "The paper applies quality assessment criteria to filter sources but then treats all included sources equally in the synthesis. A case study from a blog post and a rigorous experiment from a top venue both count as '1' in the frequency tables. This launders quality differences across sources."
     },
     {
       "flag": "Percentage denominators shift across tables",
       "detail": "Many tables note 'One paper can have more than one [category],' meaning percentages are computed over total coded instances rather than total papers. This inflates apparent coverage and makes it difficult to assess how many unique papers contribute to each finding. For example, RQ1.2 (Table 12) has 105+ model instances from 46 papers."
     },
     {
       "flag": "Missing publication bias analysis",
       "detail": "The paper does not discuss whether positive results about GenAI in software architecture are overrepresented in the literature. Given industry hype around GenAI, publication bias is a significant concern that could inflate reported benefits and understate challenges."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models for software engineering: Survey and open problems",
       "authors": ["A. Fan", "B. Gokkaya", "M. Harman", "M. Lyubarskiy", "S. Sengupta", "S. Yoo", "J. M. Zhang"],
       "year": 2023,
       "relevance": "Major survey on LLM applications across the SE lifecycle, finding emphasis on code generation with limited attention to requirements/design phases."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["X. Hou", "Y. Zhao", "Y. Liu", "Z. Yang", "K. Wang", "L. Li", "X. Luo", "D. Lo", "J. Grundy", "H. Wang"],
       "year": 2024,
       "relevance": "SLR analyzing 395 articles on LLMs in SE, concluding most applications focus on software development with limited design applications."
     },
     {
       "title": "A survey on large language models for code generation",
       "authors": ["J. Jiang", "F. Wang", "J. Shen", "S. Kim", "S. Kim"],
       "year": 2024,
       "arxiv_id": "2406.00515",
       "relevance": "Survey of LLMs specifically for code generation, developing a taxonomy and identifying research-practice gaps."
     },
     {
       "title": "Software testing with large language models: Survey, landscape, and vision",
       "authors": ["J. Wang", "Y. Huang", "C. Chen", "Z. Liu", "S. Wang", "Q. Wang"],
       "year": 2024,
       "relevance": "SLR on LLMs for software testing, analyzing 102 studies and identifying practical integration barriers."
     },
     {
       "title": "Using chatgpt in software requirements engineering: A comprehensive review",
       "authors": ["N. Marques", "R. R. Silva", "J. Bernardino"],
       "year": 2024,
       "relevance": "Comprehensive review of ChatGPT applications in requirements engineering, documenting challenges and future directions."
     },
     {
       "title": "Impacts of the usage of generative artificial intelligence on software development process",
       "authors": ["P. d. O. Santos", "A. C. Figueiredo", "P. Nuno Moura", "B. Diirr", "A. C. Alvim", "R. P. D. Santos"],
       "year": 2024,
       "relevance": "SLR on GenAI impact on SE lifecycle confirming dominance of development/testing applications."
     },
     {
       "title": "Artificial intelligence for software architecture: Literature review and the road ahead",
       "authors": ["A. Bucaioni", "M. Weyssow", "J. He", "Y. Lyu", "D. Lo"],
       "year": 2025,
       "arxiv_id": "2504.04334",
       "relevance": "SLR on AI integration with software architecture, identifying needs for real-time adaptation, automated documentation, and robust benchmarking."
     },
     {
       "title": "Software architecture meets llms: A systematic literature review",
       "authors": ["L. Schmid", "T. Hey", "M. Armbruster", "S. Corallo", "D. Fuchss", "J. Keim", "H. Liu", "A. Koziolek"],
       "year": 2025,
       "relevance": "SLR on LLMs in software architecture identifying four application areas and gaps in advanced prompting and evaluation."
     },
     {
       "title": "Application of large language models to software engineering tasks: Opportunities, risks, and implications",
       "authors": ["I. Ozkaya"],
       "year": 2023,
       "relevance": "Pragmatic overview of LLM opportunities and risks in SE, highlighting bias, data quality, privacy, and explainability challenges."
     },
     {
       "title": "Navigating the complexity of generative ai adoption in software engineering",
       "authors": ["D. Russo"],
       "year": 2024,
       "relevance": "Analysis of GenAI adoption complexity in software engineering with implications for how development practices are changing."
     },
     {
       "title": "LLMs for code: The potential, prospects, and problems",
       "authors": ["T. Sharma"],
       "year": 2024,
       "relevance": "Analysis of LLM potential and problems for code-related tasks including generation quality and maintainability concerns."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Provides a taxonomy and research roadmap but no immediately usable tools or techniques for practitioners."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that 93% of studies lack GenAI output validation is noteworthy but the overall message — that GenAI in architecture is immature — largely confirms expectations."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No novel AI risk or security concerns are raised; challenges mentioned (hallucinations, accuracy) are well-known."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims; the paper is a neutral synthesis of existing literature."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, tool, or demo is provided — only a Zenodo data archive."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from University of Oulu and University of Arizona; not famous AI labs but established SE research groups. Published in Journal of Systems and Software."
     }
   }
 }