ai-research-survey

scan.json (17755B)

---

 {
   "paper": {
     "title": "Continuous Software Engineering Practices in AI/ML Development Past the Narrow Lens of MLOps: Adoption Challenges",
     "authors": ["Sini Vänskä", "Kai-Kristian Kemell", "Tommi Mikkonen", "Pekka Abrahamsson"],
     "year": 2024,
     "venue": "e-Informatica Software Engineering Journal",
     "doi": "10.37190/e-Inf240102"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code or analysis scripts are released. No repository URL mentioned."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Interview data is not released. No data download link or supplementary materials provided."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative interview study with no computational environment to specify."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The interview instrument is provided in Appendix A, but no step-by-step instructions for reproducing the analysis are given."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative interview study with no quantitative results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative study; no statistical comparisons are made."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative study; no effect sizes applicable."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "The authors justify N=8 by citing the exploratory nature of the study and referencing Eisenhardt's recommendation for case study research on novel topics (Section 6.2)."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative study with no quantitative variance to report."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative exploratory study; no baselines applicable."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No baselines applicable for this qualitative study."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system or method to ablate; this is a qualitative interview study."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative metrics; qualitative thematic analysis."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate; the study itself is interview-based qualitative research."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No test sets; qualitative study."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by the 13 continuous SE practices and organized into 5 themes (tools/methods, business strategy, development, operations, improvement/innovation). Table 2 shows code occurrences per practice."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses challenges and failures in adopting continuous practices, including practices that were entirely absent (continuous compliance, trust, security — PEC1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative findings reported: continuous compliance/trust/security absent from data (PEC1), SE methods have little impact on ML development (PEC2), ML experts work in silos (PEC3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims challenges were identified in adopting continuous SE practices, which is supported by the 7 PECs in the results."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes observational claims about challenges and does not assert causal relationships. Language like 'results in' and 'makes it difficult' describe observed associations from interviews, not tested causal mechanisms."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 explicitly acknowledges that N=8 limits generalizability, that convenience sampling is a limitation, and that 3 of 8 organizations were research projects rather than purely industrial contexts."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The discussion section considers alternative factors: COVID-19 remote work affecting collaboration (Section 5.2), project type (external vs internal customers) affecting operations emphasis (PEC6), and domain-specific regulatory factors (PEC1 discussion)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No AI models used in this study; it is a qualitative interview study."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No AI prompting used."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No AI models or computational experiments."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the thematic analysis approach: interviews were transcribed, coded using deductive codes based on the research framework, and analyzed to identify challenges. The coding process and themes are documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 is a dedicated Limitations subsection with substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses specific threats: respondents held different roles influencing answers, only one respondent per organization limits understanding, 3 of 8 were research projects not purely industrial, and convenience sampling."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 explicitly states what was not tested: they did not focus on specific technologies or project contexts, did not study organizations that specifically claim to use MLOps, and acknowledge their findings are 'less specific' as a result."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Interview transcripts are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes data collection: thematic semi-structured interviews, conducted digitally during COVID-19, in Finnish or English, with the interview instrument provided in Appendix A."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states convenience sampling was used. Section 6.2 acknowledges this as a limitation. Table 1 lists respondent roles."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 4.1-4.2 describe the pipeline: interviews conducted → transcribed → coded using deductive codes from the framework → themes identified → PECs formulated. Table 2 shows code occurrences."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This work was partly funded by local authorities (\"Business Finland\") under grant agreement ITEA-2020-20219-IML4E of the ITEA4 programme.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Deloitte Finland, University of Helsinki, University of Jyväskylä, Tampere University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Business Finland is a public funding agency; the ITEA4 programme is a European research programme. Neither has a financial stake in the study's findings about continuous SE adoption challenges."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. One author is affiliated with Deloitte, a consulting firm, but no conflict-of-interest declaration addresses this."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model evaluated on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model evaluated on any benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model evaluated on any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration mentioned. The study involves human participants (interviewees)."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite conducting interviews with human participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 lists job titles for all 8 respondents. Section 4.1 notes experience ranged from months to decades, and that respondents came from different organizations."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "Beyond 'respondents from different organizations working on AI-related projects' selected via convenience sampling, no explicit inclusion/exclusion criteria are stated."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study with conditions; this is a qualitative interview study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study; blinding is not applicable to qualitative interviews."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information on whether any invited participants declined or dropped out. Only the final N=8 is reported."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative interview study; no computational method with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative interview study; no computational budget applicable."
       }
     }
   },
   "claims": [
     {
       "claim": "Continuous compliance, continuous trust, and continuous security were not present in the interview data from ML development organizations.",
       "evidence": "Table 2 shows zero code occurrences for these three practices across all 8 respondents (PEC1, Section 5).",
       "supported": "moderate"
     },
     {
       "claim": "SE methods used by the broader organization have little impact on ML development processes.",
       "evidence": "Multiple respondents described not following any SE method strictly; agile adoption was limited to sprints with no deeper integration (PEC2, Section 5.1).",
       "supported": "moderate"
     },
     {
       "claim": "ML experts often work in a silo, not participating in business strategy activities.",
       "evidence": "4 of 8 respondents said ML experts worked independently; 3 felt they did not understand the point of their project; most had little customer interaction (PEC3, Section 5.2).",
       "supported": "moderate"
     },
     {
       "claim": "Automated testing in ML development remains a challenge, with only 1 of 8 respondents using automated testing via MLOps pipeline.",
       "evidence": "Section 5.3 reports only Respondent 7 discussed automated testing; others relied on manual testing (PEC4).",
       "supported": "moderate"
     },
     {
       "claim": "The lack of user and customer interaction makes it difficult for ML experts to ensure continuous use of the product.",
       "evidence": "Section 5.4 reports ML experts seldom had direct contact with users; feedback only reached them if something was 'great or terribly wrong' (PEC5).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["qualitative", "case-study"],
   "key_findings": "Through interviews with 8 ML experts from different organizations, the study found that continuous SE practices beyond DevOps/MLOps are poorly adopted in ML development. Key challenges include ML experts working in silos with limited collaboration, SE methods having little impact on ML development processes, automated testing remaining difficult, and lack of user/customer interaction hindering continuous operations. Three continuous practices (compliance, trust, security) were entirely absent from the data.",
   "red_flags": [
     {
       "flag": "Very small sample size",
       "detail": "Only 8 respondents from 8 organizations, selected via convenience sampling. Three organizations were research projects rather than purely industrial contexts. The authors acknowledge this limitation but still draw broad conclusions about ML development challenges."
     },
     {
       "flag": "No ethics approval mentioned",
       "detail": "The study involves human participants (interviewees) but does not mention IRB or ethics board approval, which is standard practice for interview studies."
     }
   ],
   "cited_papers": [
     {
       "title": "Software engineering for AI-Based systems: A survey",
       "authors": ["S. Martínez-Fernández", "J. Bogner", "X. Franch", "M. Oriol", "J. Siebert"],
       "year": 2022,
       "relevance": "Comprehensive survey on SE challenges for AI-based systems, directly relevant to understanding engineering practices in AI/ML development."
     },
     {
       "title": "A software engineering perspective on engineering machine learning systems: State of the art and challenges",
       "authors": ["G. Giray"],
       "year": 2021,
       "relevance": "Surveys SE challenges in ML system development, a core topic of the survey scope."
     },
     {
       "title": "Adoption and effects of software engineering best practices in machine learning",
       "authors": ["A. Serban", "K. van der Blom", "H. Hoos", "J. Visser"],
       "year": 2020,
       "relevance": "Empirical study on SE best practice adoption rates in ML development teams."
     },
     {
       "title": "Collaboration challenges in building ML-enabled systems: Communication, documentation, engineering, and process",
       "authors": ["N. Nahar", "S. Zhou", "G. Lewis", "C. Kästner"],
       "year": 2022,
       "relevance": "Studies collaboration challenges between ML and SE teams, directly relevant to agentic AI development practices."
     },
     {
       "title": "Data scientists in software teams: State of the art and challenges",
       "authors": ["M. Kim", "T. Zimmermann", "R. DeLine", "A. Begel"],
       "year": 2017,
       "relevance": "Foundational study on integrating data science/ML roles into software teams."
     },
     {
       "title": "Towards MLOps: A framework and maturity model",
       "authors": ["M.M. John", "H.H. Olsson", "J. Bosch"],
       "year": 2021,
       "relevance": "Proposes MLOps framework and maturity model relevant to AI/ML software engineering practices."
     },
     {
       "title": "Understanding development process of machine learning systems: Challenges and solutions",
       "authors": ["E. de Souza Nascimento", "I. Ahmed", "E. Oliveira", "M.P. Palheta", "I. Steinmacher"],
       "year": 2019,
       "relevance": "Studies ML development process challenges, relevant to understanding SE practices in AI development."
     }
   ]
 }