ai-research-survey

scan.json (24912B)

---

 {
   "paper": {
     "title": "Measuring Technical Debt in AI-Based Competition Platforms",
     "authors": ["Dionysios Sklavenitis", "Dimitris Kalles"],
     "year": 2024,
     "venue": "13th Hellenic Conference on Artificial Intelligence (SETN 2024)",
     "arxiv_id": "2405.11825",
     "doi": "10.1145/3688671.3688783"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis", "qualitative"],
   "key_findings": "Through a scoping review of 72 studies, the authors identified and categorized 18 types of technical debt in AI-based systems, including algorithm, architectural, code, configuration, data, model, ethics, infrastructure, and test debt. They propose a new type — Accessibility Debt — specific to AI competition platforms, addressing barriers participants face from inadequate platform usability. A 68-question questionnaire with a 1-5 scoring system is developed for organizers and participants to assess technical debt, but it has not been empirically validated.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code, analysis scripts, or repository links are provided in the paper. The questionnaire and classification data are presented only within the paper text."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset or supplementary data files are released. The full list of 100 reviewed studies is presented in tables within the paper but not as a downloadable dataset."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or tool specifications are provided. The scoping review methodology does not describe any software tools used for analysis."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the search strategy is documented (Appendix A), there are no step-by-step reproduction instructions for replicating the full review process including screening decisions and data extraction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative scoping review with no statistical experiments or quantitative results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative quantitative claims are made that would require significance testing. The paper is a qualitative classification study."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative effects are measured. The paper classifies types of technical debt and proposes a questionnaire without empirical testing."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "As a scoping review, the sample size (number of papers) is determined by the search strategy rather than requiring statistical justification."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs or quantitative measurements are conducted that would require variance reporting."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions that 'to our knowledge, there are no comparable initiatives' and references [30] as partially related but does not conduct a structured comparison against prior surveys or frameworks for measuring technical debt."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No structured comparison with prior surveys or technical debt measurement tools is conducted. The only reference to a comparable effort ([30] Breck et al. 2017) is noted as focusing solely on ML testing/monitoring."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "There is no system with components to ablate. The paper proposes a classification taxonomy and questionnaire."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted and no metrics are evaluated. The questionnaire is proposed but not tested."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs are evaluated by humans. The questionnaire is proposed for future evaluation by organizers and participants."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No test set or experimental evaluation exists. This is a literature review with a proposed framework."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides a breakdown of 18 types of technical debt with document counts and publication spans. Table 2 shows the number of questionnaire questions per debt type. Table 6 in the appendix lists individual studies by debt type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss cases where its classification scheme might fail, borderline categorization decisions, or studies that were difficult to classify. All examples are hypothetical success scenarios."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. The paper does not discuss any classification approaches that were tried and abandoned, or types of debt that were considered but rejected."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: they identify and categorize types of technical debt (Table 1, Section 3.2), develop a questionnaire (Section 4, Appendix C), and introduce Accessibility Debt (Section 3.2.9). Claims are appropriately scoped as a framework proposal."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no empirical causal claims. Statements like 'may lead to' and 'can compromise' are qualitative assessments from the literature, not causal claims from the authors' own data."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper broadly claims the framework applies to AI-based competition platforms generally, but the questionnaire has not been tested on any actual platform. The title and abstract make broad claims ('measuring technical debt') for a framework that is entirely unvalidated. Future work acknowledges the need to test across 'various competition platforms' and 'diverse stakeholders.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative classification schemes, whether different categorizations of technical debt would be equally valid, or whether the 18 types might overlap or be reducible to fewer dimensions."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The questionnaire scores are presented as a measure of technical debt, but the paper does not discuss whether questionnaire responses actually correlate with real technical debt levels. The gap between self-reported questionnaire answers and actual technical debt presence is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No AI models are used in this study. It is a literature review with a proposed framework."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting of AI models is involved in this study."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No models or algorithms requiring hyperparameters are used."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A documents the PRISMA flow: 870 records identified → duplicates removed (n=65) → 169 screened → 85 excluded → 84 full-text assessed → 72 included. Search strings, databases, date ranges, and inclusion/exclusion criteria are specified in Sections A.3-A.8."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated discussion of internal and external validity threats. Additional limitations are discussed in Appendix A.12."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The threats section discusses generic concerns: researcher subjectivity ('other researchers who wish to use our strategy may come up with slightly different results') and that 'the area of technical debt in AI-based systems is constantly evolving.' These are standard disclaimers rather than threats specific to this study's particular findings or classification decisions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the framework does NOT cover or cannot measure. Future work implies the scope is limited to academic platforms (mentioning plans to test commercial environments like Azure and Kaggle), but no explicit scope boundaries are stated in the paper body."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw review data (screening decisions, extraction sheets, coding forms) is available for independent verification. Only aggregated results appear in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A describes the data collection in detail: five databases searched (Google Scholar, ACM, IEEE Xplore, Scopus, Springer), exact search strings, date range (2012-Feb 2024), and inclusion/exclusion criteria (I1-I3, E1-E5)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. Paper selection methods are covered under data_collection_described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The PRISMA flow diagram in Appendix A.13 documents each pipeline stage with counts: 870 records identified → duplicates removed → 169 screened → 85 excluded → 84 assessed for eligibility → 10 excluded → 74 full-text assessed → 2 excluded → 72 included. Exclusion reasons are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors are identified as affiliated with Hellenic Open University, Patras, Greece, with email addresses provided."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence of funding cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model on a benchmark. It is a scoping literature review."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation is conducted. This is a literature review and framework proposal."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is involved in this study."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The proposed questionnaire is not yet administered to participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are recruited. Inclusion/exclusion criteria for papers are described separately."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No experimental study with human participants is conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No experimental study requiring blinding is conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no system to measure inference cost for."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.13 explicitly references PRISMA guidelines and includes a PRISMA flow diagram. The methodology includes structured search strings across five databases, defined inclusion/exclusion criteria, and a two-phase screening process."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of its 72 included studies. All papers are treated equally regardless of rigor. No quality scoring rubric or risk-of-bias assessment is applied to source papers."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias is present. The paper does not consider whether its sources skew toward positive results, nor does it use funnel plots or other publication bias tests."
       }
     }
   },
   "claims": [
     {
       "claim": "18 distinct types of technical debt exist in AI-based systems, categorized from 100 reviewed studies.",
       "evidence": "Table 1 lists 18 types (Algorithm, Architectural, Build, Code, Configuration, Data, Defect, Design, Documentation, Ethics, Infrastructure, Model, People, Process, Requirements, SATD, Test, Versioning) with document counts and publication spans. Table 6 in the appendix maps individual studies to debt types. Section A.9 describes the PRISMA selection yielding 72 studies.",
       "supported": "moderate"
     },
     {
       "claim": "Accessibility Debt is a new type of technical debt specific to AI-based competition platforms.",
       "evidence": "Section 3.2.9 defines Accessibility Debt as 'barriers that participants encounter due to the lack of immediate usability of platform technologies.' A hypothetical example is provided but no empirical evidence demonstrates this is a distinct or measurable category.",
       "supported": "weak"
     },
     {
       "claim": "A 68-question questionnaire can quantify technical debt in AI-based competition platforms.",
       "evidence": "Section 4 describes the scoring methodology (1-5 scale, YES/NO/NA/Don't Know responses) and Tables 3-4 provide worked examples. The full 68 questions are listed in Appendix C. However, the questionnaire has not been tested on any platform or with any participants — validation is explicitly listed as future work (Section 7).",
       "supported": "weak"
     },
     {
       "claim": "The nine most significant types of technical debt for AI competition platforms are Algorithm, Architectural, Configuration, Data, Model, Ethics, Infrastructure, Test, and Accessibility.",
       "evidence": "Section 3.2 states these were selected 'based on their prevalence in the literature [8, 14]' and the unique relevance of Accessibility Debt to competition platforms. The selection criteria for 'most significant' are not rigorously defined.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Unvalidated measurement instrument",
       "detail": "The paper's primary contribution — a 68-question questionnaire for measuring technical debt — has not been tested on any actual platform, with any real organizers or participants. The authors explicitly state validation is future work (Section 7). Yet the title claims 'measuring' technical debt, implying an operational capability that does not yet exist."
     },
     {
       "flag": "All examples are hypothetical",
       "detail": "Every example illustrating the 18 types of technical debt in competition platforms is a hypothetical scenario, not a real case study. No actual AI competition platform was analyzed to demonstrate the framework's applicability."
     },
     {
       "flag": "No quality assessment of source papers",
       "detail": "The scoping review treats all 72-100 included papers equally without assessing their methodological quality. This risks laundering weak or preliminary findings alongside rigorous studies."
     },
     {
       "flag": "Single-author initial screening",
       "detail": "The first selection phase was conducted by a single author who read all articles and made comments. The second author reviewed only the comments, not the full papers. This introduces potential screening bias despite the described mitigation."
     },
     {
       "flag": "Questionnaire scoring lacks psychometric justification",
       "detail": "The 1-5 scoring weights assigned to questionnaire items appear arbitrary with no psychometric basis. For example, why does 'usability testing' score 5 while 'feedback mechanisms' scores 3? No weighting methodology, expert panel, or calibration is described."
     }
   ],
   "cited_papers": [
     {
       "title": "Hidden technical debt in machine learning systems",
       "authors": ["D. Sculley", "G. Holt", "D. Golovin", "E. Davydov", "T. Phillips", "D. Ebner"],
       "year": 2015,
       "relevance": "Foundational work introducing the concept of hidden technical debt in ML systems, including boundary erosion, entanglement, and feedback loops."
     },
     {
       "title": "Is using deep learning frameworks free? Characterizing technical debt in deep learning frameworks",
       "authors": ["J. Liu", "Q. Huang", "X. Xia", "E. Shihab", "D. Lo", "S. Li"],
       "year": 2020,
       "doi": "10.1145/3377815.3381377",
       "relevance": "Identifies extensive technical debt in deep learning frameworks across algorithm, build, defect, and SATD categories."
     },
     {
       "title": "Characterizing Technical Debt and Antipatterns in AI-Based Systems: A Systematic Mapping Study",
       "authors": ["J. Bogner", "R. Verdecchia", "I. Gerostathopoulos"],
       "year": 2021,
       "doi": "10.1109/TechDebt52882.2021.00016",
       "relevance": "Systematic mapping of 72 antipatterns and 46 solutions for technical debt in AI systems, major source for the paper's classification."
     },
     {
       "title": "Software Engineering for AI-Based Systems: A Survey",
       "authors": ["S. Martínez-Fernández", "J. Bogner", "X. Franch", "M. Oriol", "J. Siebert", "A. Trendowicz", "S. Wagner"],
       "year": 2022,
       "doi": "10.1145/3487043",
       "relevance": "Comprehensive survey synthesizing SE practices for AI development, examining integration of software engineering in AI."
     },
     {
       "title": "Software Engineering for Machine Learning: A Case Study",
       "authors": ["S. Amershi", "A. Begel", "C. Bird", "R. DeLine", "H. Gall", "E. Kamar", "N. Nagappan"],
       "year": 2019,
       "doi": "10.1109/ICSE-SEIP.2019.00042",
       "relevance": "Industry case study examining SE challenges in ML development at Microsoft, providing practical perspective on AI technical debt."
     },
     {
       "title": "The ML test score: A rubric for ML production readiness and technical debt reduction",
       "authors": ["E. Breck", "S. Cai", "E. Nielsen", "M. Salib", "D. Sculley"],
       "year": 2017,
       "doi": "10.1109/BigData.2017.8258038",
       "relevance": "Proposes a rubric for ML production readiness and debt reduction — the closest prior work to this paper's questionnaire approach."
     },
     {
       "title": "Code and Architectural Debt in Artificial Intelligence-Enabled Systems: On the Prevalence, Severity, Impact, and Management Strategies",
       "authors": ["G. Recupito", "F. Pecorelli", "G. Catolino", "V. Lenarduzzi", "D. Taibi"],
       "year": 2024,
       "relevance": "Studies prevalence and management of code and architectural debt specifically in AI systems."
     },
     {
       "title": "Code Smells in Machine Learning Systems",
       "authors": ["J. Gesi", "S. Liu", "J. Li", "I. Ahmed", "N. Nagappan", "D. Lo", "L. Bao"],
       "year": 2022,
       "relevance": "Identifies five new code smells affecting maintainability of deep learning systems, contributing to understanding of code debt in AI."
     },
     {
       "title": "CodaLab Competitions: An open source platform to organize scientific challenges",
       "authors": ["A. Pavao"],
       "year": 2022,
       "relevance": "Proposes using CodaLab to manage technical debt in competitive AI settings, directly relevant to AI competition platform quality."
     },
     {
       "title": "Software Quality for AI: Where We Are Now?",
       "authors": ["V. Lenarduzzi", "F. Lomio", "S. Moreschini", "D. Taibi", "D. A. Tamburri"],
       "year": 2021,
       "doi": "10.1007/978-3-030-65854-0_4",
       "relevance": "Examines the evolving nature of software quality metrics for AI systems, highlighting that AI systems are 'just like traditional software until they're not.'"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The questionnaire framework could be useful to AI competition organizers, but it is entirely unvalidated and requires significant adaptation before practical use."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "The findings confirm expected challenges with technical debt in AI systems; no conventional wisdom is challenged."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns are raised; the paper addresses software engineering quality, not risks."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict is present in the paper's findings or framing."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, tool, or demo is available; the questionnaire exists only as text in the appendix."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors are from Hellenic Open University; the venue (SETN) is a regional conference with limited international visibility."
     }
   }
 }