ai-research-survey

scan.json (23318B)

---

 {
   "paper": {
     "title": "Bridging MDE and AI: A Systematic Review of Domain-Specific Languages and Model-Driven Practices in AI Software Systems Engineering",
     "authors": [
       "Simon Rädler",
       "Luca Berardinelli",
       "Karolin Winter",
       "Abbas Rahimi",
       "Stefanie Rinderle-Ma"
     ],
     "year": 2023,
     "venue": "SoSyM Journal (preprint)",
     "arxiv_id": "2307.04599",
     "doi": null
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper references an online repository at https://github.com/sraedler/Model-Driven-Engineering4Artificial-Intelligence containing the SLR protocol spreadsheet and bibliographic entries (Section 3.2, footnote 3)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The search terms, results, and extracted data are archived and available online at the GitHub repository referenced in footnote 3. The SLR protocol execution is documented in a spreadsheet available online."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or software dependency specifications are provided. This is an SLR so there is no computational environment per se, but the analysis tools/scripts (if any) are not documented."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the SLR protocol is described in Section 3 and the search terms are given, there are no step-by-step reproduction instructions explaining how to use the online artifacts to replicate the study end-to-end."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not perform statistical experiments or report quantitative measurements requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not make comparative claims requiring statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review without meta-analysis or statistical aggregation. No effect sizes are applicable."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review. The 'sample size' (18 primary studies) is a result of the search/selection protocol, not a pre-determined sample requiring power analysis."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review without quantitative experiments. No variance or standard deviation is applicable."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares its scope and approach against prior surveys and secondary studies by Portugal et al. [68], Naveed et al. [64], Giray [38], Martínez-Fernández et al. [59], and Bucaioni et al. [18] in the Related Work section (Section 2.3), explaining how their work differs and complements these."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The related surveys compared against include Naveed et al. [64] (2024), Martínez-Fernández et al. [59] (2022), Giray [38] (2021), and Bucaioni et al. [18] (2022), which are contemporary works."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review. There is no system with components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review, not a system evaluation. There are no performance metrics to report."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that classifies and discusses published studies. Human evaluation of system outputs is not relevant."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review with no test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides detailed per-study breakdowns across MDE concerns (Table 5), AI concerns (Table 5), frameworks/tools (Table 7), and artifact availability/application domains (Table 8). Each of the 18 studies is individually assessed."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses gaps and weaknesses in the reviewed approaches. Section 5 (Discussion) and Section 5.6 (RQ6) explicitly identify shortcomings: lack of business understanding support, incomplete CRISP-DM coverage, case-specificity, lack of user studies, and missing runtime support."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.6 reports several negative findings: no low-code/no-code contributions found for intelligent systems, lack of closed-loop processes, limited general applicability, and few user studies conducted by the reviewed approaches."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: (1) language workbenches are paramount (Table 5 shows 17/18 use LWs), (2) most prominent AI concerns are training and modeling (14/18 for Model Training in Table 5), (3) data preparation gets minor emphasis (8/18 for Feature Preparation), (4) business understanding is rarely reflected (5/18). All supported by data in Tables 5-8."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a systematic literature review that describes the state of the art. It does not make causal claims about why certain outcomes occur, only descriptive observations about what exists."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds its scope to MDE4AI (not AI4MDE), to DSLs with AI extensions as first-class entities, and acknowledges the limited volume of publications found (Section 5). The threats to validity section (Section 6) discusses limitations of generalizability."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses that the limited volume of publications may result from their selective exclusion criteria and the focus on MDE4AI excluding the complementary AI4MDE perspective. Section 6 (Threats to Validity) discusses how interdisciplinary terminology and various definitions could affect findings."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review. No AI models are used in the research methodology."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not use prompting or LLMs."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review with no computational experiments requiring hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a traditional systematic literature review."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents the filtering pipeline in detail: 1335 candidate studies from 5 databases, filtered to 130 by IC/EC on title/abstract, then to 18 after full-text reading, plus 4 from snowballing (Section 3.3, Figure 1, Table 2). The inclusion and exclusion criteria are explicitly stated in Table 2 with detailed explanations."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated section discussing construct validity, internal validity, external validity, and conclusion validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats to validity section includes specific concerns: at least two researchers independently analyze each study with a third for disagreements (construct validity), the interdisciplinary nature of MDE and AI fields may cause terminology issues (conclusion validity), and the subjective nature of data extraction is mitigated by the protocol. These are specific to this study rather than purely generic."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: MDE4AI only (not AI4MDE), excludes DSLs without first-class AI concepts, excludes non-primary studies, vision papers, and theses (Table 2). Section 3.4 notes that deployment phase of CRISP-DM is beyond scope. The related work section (Section 2.3) explicitly differentiates the scope from prior surveys."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The raw search results, selection spreadsheet, and bibliographic entries are available at the GitHub repository (footnote 3: https://github.com/sraedler/Model-Driven-Engineering4Artificial-Intelligence)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the search process in detail: 5 databases, keyword sets S1 (12 MDE terms) and S2 (10 AI terms) combined into 120 conjunctive propositions, executed as individual queries. The automated search was run in November 2022 and February 2024."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The paper is a systematic literature review analyzing published studies, not conducting research with human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 and Section 3 document the full pipeline: 1335 collected from queries -> IC/EC applied to titles/abstracts -> 130 papers -> full-text reading -> 18 papers retained + 4 from snowballing. The data extraction template (Table 4) details what was extracted from each paper."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the footnote on page 1: 'This project has been partially supported and funded by the AIDOaRt project, an ECSEL Joint Undertaking (JU) under grant agreement No. 101007350 and the Austrian Research Promotion Agency (FFG) via the Austrian Competence Center for Digital Production (CDP) under the contract number 881843.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Technical University of Munich, Technical University of Vienna, Johannes Kepler University, and Eindhoven University of Technology. One author (Rädler) is both first author and author of primary study [71], which is acknowledged."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders are the ECSEL Joint Undertaking (EU program) and the Austrian Research Promotion Agency via a competence center. These are public research funding bodies that do not have a financial stake in the specific outcomes of this systematic review."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not evaluate any pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not evaluate any pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a systematic literature review."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a systematic literature review."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a systematic literature review."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a systematic literature review. (Note: the paper does have inclusion/exclusion criteria for papers, addressed under data_preprocessing_documented.)"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a systematic literature review."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a systematic literature review."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a systematic literature review."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review. No computational method with inference costs is proposed."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review. No significant computational resources are required."
       }
     }
   },
   "claims": [
     {
       "claim": "Language workbenches are of paramount importance in dealing with all aspects of modeling language development and are leveraged in 17 of 18 selected studies.",
       "evidence": "Table 5 shows the Language Workbench (LW) row has a filled square for 17 of 18 approaches, with only [42] lacking a specific LW implementation.",
       "supported": "strong"
     },
     {
       "claim": "The most prominent AI-related concern addressed by MDE approaches is model training (14/18 studies), while data preparation receives less emphasis (8/18).",
       "evidence": "Table 5 quantifies AI concerns: Model Training (MTrain) is supported by 14 studies, Feature Preparation (FP) by 8, Business Understanding (BU) by only 5.",
       "supported": "strong"
     },
     {
       "claim": "Early project phases supporting interdisciplinary communication (CRISP-DM Business Understanding) are rarely reflected, with only 5 of 18 approaches supporting it.",
       "evidence": "Table 5 shows Business Understanding (BU) supported by 5 studies. Section 5.5 discusses the limited communication support.",
       "supported": "strong"
     },
     {
       "claim": "The use of MDE for AI is still in its early stages with no single widely-used tool or method.",
       "evidence": "Section 5 and 7 discuss the fragmented landscape. Table 7 shows a wide variety of tools with no dominant one. Only 18 primary studies were found from 1335 candidates, and publication volumes are limited (Figure 2).",
       "supported": "strong"
     },
     {
       "claim": "Textual concrete syntax is preferred over graphical syntax in the reviewed MDE4AI approaches.",
       "evidence": "Table 5 shows 10 textual, 7 graphical, 2 XMI concrete syntaxes. Section 5.1 discusses this preference.",
       "supported": "strong"
     },
     {
       "claim": "No valuable contributions were identified for low-code/no-code development platforms targeting intelligent AI systems.",
       "evidence": "Section 5.6 states: 'this study did not identify any valuable contributions, despite the recent trends in low-code and no-code development platforms targeting intelligent systems.'",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "meta-analysis"
   ],
   "key_findings": "This systematic literature review of 18 primary studies (from 1335 candidates) finds that MDE approaches for AI software systems predominantly focus on the model training phase of CRISP-DM, while time-consuming tasks like data preparation and early business understanding phases are underserved. Language workbenches are nearly universally adopted (17/18 studies), with EMF and Xtext being the most common tooling, and textual syntax preferred over graphical. The field is fragmented with no dominant tool or method, and current approaches tend to be case-specific rather than generally applicable.",
   "red_flags": [
     {
       "flag": "Author self-inclusion",
       "detail": "First author Simon Rädler is also an author of primary study [71] (Raedler et al.), which is one of the 18 reviewed papers. While this is acknowledged in the text, it represents a potential bias in the review process."
     },
     {
       "flag": "No structured quality assessment of primary studies",
       "detail": "While the paper mentions using quality criteria from [56] (CASP tool), no quality scores or structured quality assessment results are presented for the 18 primary studies. The reader cannot assess the methodological rigor of the individual reviewed papers."
     },
     {
       "flag": "Small corpus",
       "detail": "Only 18 primary studies were retained from 1335 candidates, which is a very aggressive filtering ratio (1.3%). While this may reflect the narrow scope, it limits the generalizability of the findings about the MDE4AI field."
     }
   ],
   "cited_papers": [
     {
       "title": "Software engineering for ai-based systems: a survey",
       "authors": ["Silverio Martínez-Fernández", "Justus Bogner", "Xavier Franch", "Marc Oriol", "Julien Siebert", "Adam Trendowicz", "Anna Maria Vollmer", "Stefan Wagner"],
       "year": 2022,
       "relevance": "Comprehensive survey on software engineering practices for AI-based systems, directly relevant to understanding SE4AI methodology and quality."
     },
     {
       "title": "A software engineering perspective on engineering machine learning systems: State of the art and challenges",
       "authors": ["Görkem Giray"],
       "year": 2021,
       "relevance": "Systematic review of 141 studies on SE for ML systems, finding no mature tools/techniques and identifying challenges in ML systems engineering."
     },
     {
       "title": "Model driven engineering for machine learning components: A systematic literature review",
       "authors": ["Hira Naveed", "Chetan Arora", "Hourieh Khalajzadeh", "John Grundy", "Omar Haggag"],
       "year": 2024,
       "relevance": "Parallel systematic review on MDE4ML selecting 46 primary studies, providing a complementary viewpoint to this paper's analysis."
     },
     {
       "title": "Modelling in low-code development: a multi-vocal systematic review",
       "authors": ["Alessio Bucaioni", "Antonio Cicchetti", "Federico Ciccozzi"],
       "year": 2022,
       "relevance": "Multi-vocal review on low-code development platforms and MDE's role, relevant to understanding AI-augmented software development tooling."
     },
     {
       "title": "On the Engineering of AI-Powered Systems",
       "authors": ["Evgeny Kusmenko", "Svetlana Pavlitskaya", "Bernhard Rumpe", "Sebastian Stuber"],
       "year": 2019,
       "relevance": "Presents DSL-based MDE approach for deep learning using MontiAnna framework, directly relevant to AI-powered system engineering methodology."
     },
     {
       "title": "A model-driven approach to machine learning and software modeling for the IoT",
       "authors": ["Armin Moin", "Moharram Challenger", "Atta Badii", "Stephan Günnemann"],
       "year": 2022,
       "relevance": "MDE approach extending ThingML for IoT with ML/data analytics, demonstrating model-driven code generation for AI in cyber-physical systems."
     },
     {
       "title": "The next Evolution of MDE: A Seamless Integration of Machine Learning into Domain Modeling",
       "authors": ["Thomas Hartmann", "Assaad Moawad", "Francois Fouquet", "Yves Le Traon"],
       "year": 2019,
       "relevance": "Proposes integration of machine learning micro-learning units into domain modeling, one of only approaches supporting runtime models."
     },
     {
       "title": "AIoTML: A Unified Modeling Language for AIoT-Based Cyber-Physical Systems",
       "authors": ["Ming Hu", "E. Cao", "Hongbing Huang", "Min Zhang", "Xiaohong Chen", "Mingsong Chen"],
       "year": 2023,
       "relevance": "Novel DSL extending ThingML for AI-IoT cyber-physical systems, relevant to understanding model-driven approaches for AI system development."
     },
     {
       "title": "Failure of AI projects: Understanding the critical factors",
       "authors": ["Jens Westenberger", "Kajetan Schuler", "Dennis Schlegel"],
       "year": 2022,
       "relevance": "Identifies critical factors in AI project failures including communication challenges, providing context for why MDE4AI approaches are needed."
     },
     {
       "title": "DescribeML: A tool for describing machine learning datasets",
       "authors": ["Joan Giner-Miguelez", "Abel Gómez", "Jordi Cabot"],
       "year": 2022,
       "relevance": "Proposes a DSL for ML dataset description using Langium, relevant to dataset documentation and reproducibility in AI development."
     }
   ]
 }