scan.json (18747B)
1 { 2 "paper": { 3 "title": "AI in Software Engineering: A Survey on Project Management Applications", 4 "authors": ["Talia Crawford", "Scott Duong", "Richard Fueston", "Ayorinde Lawani", "Samuel Owoade", "Abel Uzoka", "Reza M. Parizi", "Abbas Yazdinejad"], 5 "year": 2023, 6 "venue": "arXiv", 7 "arxiv_id": "2307.15224", 8 "doi": "10.48550/arXiv.2307.15224" 9 }, 10 "checklist": { 11 "artifacts": { 12 "code_released": { 13 "applies": true, 14 "answer": false, 15 "justification": "No source code, analysis scripts, or repository URLs are provided anywhere in the paper." 16 }, 17 "data_released": { 18 "applies": true, 19 "answer": false, 20 "justification": "No dataset or structured extraction of surveyed works is released. The survey data is only presented in the narrative and Table I within the paper itself." 21 }, 22 "environment_specified": { 23 "applies": false, 24 "answer": false, 25 "justification": "This is a narrative literature survey with no computational experiments, so environment specifications are structurally inapplicable." 26 }, 27 "reproduction_instructions": { 28 "applies": true, 29 "answer": false, 30 "justification": "No reproduction instructions are provided. There is no description of search queries, databases used, or systematic methodology that would allow someone to replicate the survey process." 31 } 32 }, 33 "statistical_methodology": { 34 "confidence_intervals_or_error_bars": { 35 "applies": false, 36 "answer": false, 37 "justification": "This is a narrative literature survey that does not perform any quantitative analysis or report numerical results." 38 }, 39 "significance_tests": { 40 "applies": false, 41 "answer": false, 42 "justification": "No experiments or quantitative comparisons are conducted in this survey." 43 }, 44 "effect_sizes_reported": { 45 "applies": false, 46 "answer": false, 47 "justification": "No quantitative analysis is performed; this is a narrative review of existing literature." 48 }, 49 "sample_size_justified": { 50 "applies": false, 51 "answer": false, 52 "justification": "No experiments are conducted. The choice of 11 papers is not justified with any formal sample size reasoning, but statistical sample size justification is not applicable to a narrative survey." 53 }, 54 "variance_reported": { 55 "applies": false, 56 "answer": false, 57 "justification": "No quantitative experiments are conducted in this paper." 58 } 59 }, 60 "evaluation_design": { 61 "baselines_included": { 62 "applies": true, 63 "answer": false, 64 "justification": "The survey does not compare itself against prior surveys or reviews on the same topic. No baseline comparison is made." 65 }, 66 "baselines_contemporary": { 67 "applies": true, 68 "answer": false, 69 "justification": "No baselines or prior surveys are compared against, so contemporaneity cannot be assessed." 70 }, 71 "ablation_study": { 72 "applies": false, 73 "answer": false, 74 "justification": "This is a literature survey with no system or components to ablate." 75 }, 76 "multiple_metrics": { 77 "applies": false, 78 "answer": false, 79 "justification": "No evaluation metrics are used; this is a narrative review." 80 }, 81 "human_evaluation": { 82 "applies": false, 83 "answer": false, 84 "justification": "This is a literature survey that does not produce outputs requiring human evaluation." 85 }, 86 "held_out_test_set": { 87 "applies": false, 88 "answer": false, 89 "justification": "No experimental evaluation is conducted." 90 }, 91 "per_category_breakdown": { 92 "applies": true, 93 "answer": true, 94 "justification": "Table I provides a per-paper breakdown comparing year, type of publication, and limitations for each of the surveyed works." 95 }, 96 "failure_cases_discussed": { 97 "applies": true, 98 "answer": true, 99 "justification": "Section IV (Challenges) discusses where AI applications in software engineering face difficulties, including accuracy issues, ethical concerns, trust problems, and agile methodology challenges." 100 }, 101 "negative_results_reported": { 102 "applies": true, 103 "answer": true, 104 "justification": "The paper discusses challenges and limitations in Section IV, including how AI adoption can lead to developer over-reliance, loss of coding ability, and failed AI chatbot experiments." 105 } 106 }, 107 "claims_and_evidence": { 108 "abstract_claims_supported": { 109 "applies": true, 110 "answer": true, 111 "justification": "The abstract claims the paper reviews eleven publications on AI in software engineering, compares them, and discusses challenges and future research. Sections II-VI deliver on these claims." 112 }, 113 "causal_claims_justified": { 114 "applies": false, 115 "answer": false, 116 "justification": "The paper does not make causal claims. It summarizes existing literature and discusses potential future applications without asserting causal relationships." 117 }, 118 "generalization_bounded": { 119 "applies": true, 120 "answer": false, 121 "justification": "The paper makes broad claims about AI in software engineering based on only 11 sources without bounding generalizations. For example, Section VII states 'AI and ML have the potential to become invaluable to software engineering' based on a small, non-systematic sample." 122 }, 123 "alternative_explanations_discussed": { 124 "applies": true, 125 "answer": false, 126 "justification": "The paper does not discuss alternative interpretations of the surveyed findings. It presents a single narrative about AI's potential without considering that the surveyed papers may be overly optimistic or that AI may not deliver the claimed benefits." 127 } 128 }, 129 "setup_transparency": { 130 "model_versions_specified": { 131 "applies": false, 132 "answer": false, 133 "justification": "No AI/ML models are used or evaluated in this survey paper." 134 }, 135 "prompts_provided": { 136 "applies": false, 137 "answer": false, 138 "justification": "No prompting of AI models is performed in this paper." 139 }, 140 "hyperparameters_reported": { 141 "applies": false, 142 "answer": false, 143 "justification": "No models are trained or used; this is a literature survey." 144 }, 145 "scaffolding_described": { 146 "applies": false, 147 "answer": false, 148 "justification": "No agentic scaffolding is used in this paper." 149 }, 150 "data_preprocessing_documented": { 151 "applies": true, 152 "answer": false, 153 "justification": "The paper does not describe how the 11 surveyed papers were selected. No search strategy, databases queried, search terms, inclusion/exclusion criteria, or filtering pipeline is documented. The selection appears ad hoc." 154 } 155 }, 156 "limitations_and_scope": { 157 "limitations_section_present": { 158 "applies": true, 159 "answer": false, 160 "justification": "There is no dedicated limitations or threats-to-validity section for the survey itself. Table I lists limitations of the surveyed works, but the survey does not discuss its own methodological limitations." 161 }, 162 "threats_to_validity_specific": { 163 "applies": true, 164 "answer": false, 165 "justification": "No threats to validity are discussed for the survey methodology. The paper does not acknowledge that reviewing only 11 papers may be insufficient, that the selection may be biased, or that the narrative approach lacks systematic rigor." 166 }, 167 "scope_boundaries_stated": { 168 "applies": true, 169 "answer": false, 170 "justification": "The paper does not explicitly state what it does NOT cover. The scope is implicitly project management applications, but boundaries are not formally defined, and the paper drifts into blockchain, IoT, and cybersecurity without clear scope justification." 171 } 172 }, 173 "data_integrity": { 174 "raw_data_available": { 175 "applies": true, 176 "answer": false, 177 "justification": "No raw data or structured extraction of the surveyed papers is made available for verification." 178 }, 179 "data_collection_described": { 180 "applies": true, 181 "answer": false, 182 "justification": "The paper does not describe how the 11 papers were found or selected. There is no mention of databases, search queries, or time periods for the literature search." 183 }, 184 "recruitment_methods_described": { 185 "applies": false, 186 "answer": false, 187 "justification": "No human participants are involved; this is a literature survey. The surveyed papers are from public sources (standard benchmark for a survey), but the selection methodology is a separate concern addressed under data_collection_described." 188 }, 189 "data_pipeline_documented": { 190 "applies": true, 191 "answer": false, 192 "justification": "No pipeline from literature search to final paper selection is documented. The paper provides no information about how papers were identified, screened, or selected for inclusion." 193 } 194 }, 195 "conflicts_of_interest": { 196 "funding_disclosed": { 197 "applies": true, 198 "answer": false, 199 "justification": "No funding source is disclosed anywhere in the paper. There is no acknowledgments section mentioning grants or sponsors." 200 }, 201 "affiliations_disclosed": { 202 "applies": true, 203 "answer": true, 204 "justification": "Author affiliations are clearly stated: Kennesaw State University (Department of Software Engineering and Game Development) and University of Guelph (Cyber Science Lab). Authors include both students and faculty." 205 }, 206 "funder_independent_of_outcome": { 207 "applies": true, 208 "answer": false, 209 "justification": "No funding is disclosed, so independence of funding cannot be assessed. The absence of a funding disclosure is itself a gap." 210 }, 211 "financial_interests_declared": { 212 "applies": true, 213 "answer": false, 214 "justification": "No competing interests or financial interests statement is present in the paper." 215 } 216 }, 217 "contamination": { 218 "training_cutoff_stated": { 219 "applies": false, 220 "answer": false, 221 "justification": "This survey does not evaluate any pre-trained model on a benchmark." 222 }, 223 "train_test_overlap_discussed": { 224 "applies": false, 225 "answer": false, 226 "justification": "No model evaluation is performed; contamination is not applicable." 227 }, 228 "benchmark_contamination_addressed": { 229 "applies": false, 230 "answer": false, 231 "justification": "No benchmark evaluation is conducted." 232 } 233 }, 234 "human_studies": { 235 "pre_registered": { 236 "applies": false, 237 "answer": false, 238 "justification": "No human participants are involved in this literature survey." 239 }, 240 "irb_or_ethics_approval": { 241 "applies": false, 242 "answer": false, 243 "justification": "No human participants are involved." 244 }, 245 "demographics_reported": { 246 "applies": false, 247 "answer": false, 248 "justification": "No human participants are involved." 249 }, 250 "inclusion_exclusion_criteria": { 251 "applies": false, 252 "answer": false, 253 "justification": "No human participants are involved." 254 }, 255 "randomization_described": { 256 "applies": false, 257 "answer": false, 258 "justification": "No human participants or experimental conditions are involved." 259 }, 260 "blinding_described": { 261 "applies": false, 262 "answer": false, 263 "justification": "No human participants or experimental conditions are involved." 264 }, 265 "attrition_reported": { 266 "applies": false, 267 "answer": false, 268 "justification": "No human participants are involved." 269 } 270 }, 271 "cost_and_practicality": { 272 "inference_cost_reported": { 273 "applies": false, 274 "answer": false, 275 "justification": "This is a survey paper with no computational method of its own." 276 }, 277 "compute_budget_stated": { 278 "applies": false, 279 "answer": false, 280 "justification": "This is a survey paper with no computational experiments." 281 } 282 } 283 }, 284 "claims": [ 285 { 286 "claim": "AI holds immense potential in software engineering, particularly in project management and planning.", 287 "evidence": "The abstract and Sections I-II summarize eleven publications that propose or discuss AI use in software project management. No quantitative evidence for the claim's magnitude is provided.", 288 "supported": "weak" 289 }, 290 { 291 "claim": "If AI is successfully implemented in software project development decisions, there could be a decrease in project failures and errors.", 292 "evidence": "Section I states this as a potential benefit. No empirical evidence is presented in the survey itself; the claim rests on the reviewed papers' proposals, most of which are themselves unvalidated frameworks.", 293 "supported": "unsupported" 294 }, 295 { 296 "claim": "Software engineers have been identifying the need for AI support in project management for over 30 years.", 297 "evidence": "Section II reviews two sources from 1992 (Partridge [7] and Bennett et al. [8]) that discuss AI use in software engineering, establishing a 30-year history of interest.", 298 "supported": "moderate" 299 }, 300 { 301 "claim": "Most companies utilize a single risk assessment of an AI model and then move on, which is not usually sufficient for a continuously learning AI system.", 302 "evidence": "Section IV attributes this to Lu et al. [10]. The claim is presented as a finding from a single study with no independent corroboration.", 303 "supported": "weak" 304 } 305 ], 306 "methodology_tags": ["meta-analysis", "qualitative"], 307 "key_findings": "This narrative literature survey reviews 11 publications (1992-2022) on AI applications in software engineering project management. The paper identifies potential benefits of AI in risk assessment, cost estimation, and requirement allocation, but most surveyed works are proposals or frameworks without empirical validation. The survey also discusses challenges including AI accuracy, ethical concerns, developer over-reliance, and trust issues. The paper does not follow systematic review methodology and provides no structured quality assessment of the surveyed works.", 308 "red_flags": [ 309 { 310 "flag": "Non-systematic survey methodology", 311 "detail": "The paper reviews only 11 papers with no described search strategy, databases, search terms, inclusion/exclusion criteria, or quality assessment. The selection appears ad hoc, making the survey unreproducible." 312 }, 313 { 314 "flag": "Scope drift into unrelated topics", 315 "detail": "Section V discusses blockchain, IoT, and AI-driven cybersecurity — topics loosely connected to the stated focus on AI in software engineering project management. Many of the references in this section are from the same author group (Yazdinejad et al.), suggesting self-citation padding." 316 }, 317 { 318 "flag": "Excessive self-citation", 319 "detail": "Of 29 references, at least 12 include author Abbas Yazdinejad (a co-author of this survey) as a co-author. Many of these citations (e.g., [13]-[29]) are for blockchain, IoT, and cybersecurity papers that are only tangentially related to the survey topic." 320 }, 321 { 322 "flag": "No quality assessment of surveyed works", 323 "detail": "The survey summarizes each paper's content and limitations but performs no structured quality assessment. This risks laundering weak proposals and unvalidated frameworks as evidence for AI's potential in software engineering." 324 }, 325 { 326 "flag": "Claims outrun evidence", 327 "detail": "The paper makes sweeping claims about AI's potential in software engineering based primarily on unvalidated framework proposals and conference talks, not on empirical evidence of AI effectiveness in project management." 328 } 329 ], 330 "cited_papers": [ 331 { 332 "title": "Towards Effective AI-powered Agile Project Management", 333 "authors": ["H. K. Dam", "T. Tran", "J. Grundy", "A. Ghose", "Y. Kamei"], 334 "year": 2019, 335 "relevance": "Proposes an AI framework for agile project management, directly relevant to AI-assisted software development processes." 336 }, 337 { 338 "title": "Artificial Neural Network Architecture and Orthogonal Arrays in Estimation of Software Projects Efforts", 339 "authors": ["N. Rankovic", "D. Rankovic", "M. Ivanovic", "L. Lazic"], 340 "year": 2021, 341 "relevance": "Proposes neural network models for software effort estimation, relevant to AI-assisted project planning." 342 }, 343 { 344 "title": "Machine Learning for Software Engineering: Models, Methods, and Applications", 345 "authors": ["K. Meinke", "A. Bennaceur"], 346 "year": 2018, 347 "relevance": "Reviews ML applications in software engineering, relevant to the survey's scope of AI in SE." 348 }, 349 { 350 "title": "Software Engineering for Machine-Learning Applications: The Road Ahead", 351 "authors": ["F. Khomh", "B. Adams", "J. Cheng", "M. Fokaefs", "G. Antoniol"], 352 "year": 2018, 353 "relevance": "Discusses challenges of integrating ML into software engineering workflows." 354 }, 355 { 356 "title": "Software Engineering for Responsible AI: An Empirical Study and Operationalised Patterns", 357 "authors": ["Q. Lu", "L. Zhu", "X. Xu", "J. Whittle", "D. Douglas", "C. Sanderson"], 358 "year": 2022, 359 "relevance": "Empirical study on ethical AI practices in software engineering, relevant to responsible AI development." 360 }, 361 { 362 "title": "Analysis of Software Engineering for Agile Machine Learning Projects", 363 "authors": ["K. Singla", "J. Bose", "C. Naik"], 364 "year": 2018, 365 "relevance": "Studies agile methodology challenges specific to ML projects, relevant to AI-assisted development practices." 366 }, 367 { 368 "title": "The Future of Software Engineering by 2050s: Will AI Replace Software Engineers?", 369 "authors": ["L. M. Abu Zohair"], 370 "year": 2018, 371 "relevance": "Survey of software engineering professionals on the future impact of AI on their roles." 372 } 373 ] 374 }