ai-research-survey

scan.json (25671B)

---

 {
   "paper": {
     "title": "Human-Centered AI Product Prototyping with No-Code AutoML: Conceptual Framework, Potentials and Limitations",
     "authors": ["Mario Truss", "Marc Schmitt"],
     "year": 2024,
     "venue": "International Journal of Human-Computer Interaction",
     "arxiv_id": "2402.07933",
     "doi": "10.1080/10447318.2024.2425454"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": ["case-study", "qualitative"],
   "key_findings": "The paper proposes a conceptual framework for integrating no-code AutoML into the AI product prototyping process, targeting AI non-experts. A single case study using Google Vertex AI for customer support ticket classification (3,440 examples) demonstrates that NC AutoML can support ideation, prototyping, and testing stages while providing insights into viability, feasibility, usability, and desirability. The authors identify limitations including inability to handle complex data problems, black-box model opacity, and restricted UI testing capabilities.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or repository is released. The paper presents a conceptual framework and a case study walkthrough of Google AutoML but provides no code artifacts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The case study uses a dataset of 3,440 customer support training examples, but this dataset is not released or made publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. The paper only mentions using Google Vertex AI without version information or technical setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The case study is described narratively but lacks sufficient detail for replication (e.g., exact AutoML configuration, data format specifics)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "The paper presents no quantitative experiments by the authors. The ML metrics shown (precision, recall, F1) are screenshots from the Google AutoML UI, not the paper's own experimental results requiring uncertainty quantification."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no comparative quantitative claims requiring significance tests. The evaluation is entirely qualitative and criteria-based."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative measurements are reported by the authors that would require effect size reporting. The paper's contribution is a conceptual framework evaluated qualitatively."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses a single case study with one NC AutoML tool, one dataset (3,440 examples), and one use case (classification) but does not justify why this sample is sufficient for the claims made about the framework's general utility."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs are conducted by the authors. The single case study walkthrough does not involve repeated measurements."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses existing prototyping approaches in Section 2.5 (Wizard of Oz, LLM prompting, design thinking) but does not perform a head-to-head comparison or systematic baseline evaluation. The criteria-based evaluation in Section 4.4 is self-referential."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included in the evaluation, so contemporaneity cannot be assessed."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The framework is a single conceptual artifact without separable components that could be ablated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses qualitative criteria (A1-A3, B1-B4, C1-C4) but no quantitative metrics. The ML metrics (precision, recall, F1) shown in the case study are from Google AutoML's output, not the paper's evaluation of its own contribution."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the framework supports AI non-experts but includes no user study or human evaluation of the framework itself. The case study was conducted by the authors, not independent AI non-experts."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not perform quantitative evaluation with training/test splits for its own contribution. The 80/20 split mentioned is internal to Google AutoML's model training in the case study."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 provides a detailed breakdown of evaluation results across integration in prototyping stages (A1-A3), product success criteria (B1-B4), and challenges addressed (C1-C4), with separate discussion for each criterion."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses specific limitations: NC AutoML cannot handle complex data acquisition, custom model development, or full UI integration. Section 5.3 discusses ethical risks. Section 5.4 lists research limitations including single tool, single dataset, single use case."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple things that did not work well: black-box nature of AutoML models, inability to test holistic AI experience without coding, limited data engineering capabilities, and inability to fully validate human-centricity (Section 5.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are reasonably hedged: 'highlighting its efficacy in supporting AI non-experts and streamlining decision-making and its limitations.' The paper does present both positive and negative findings in Section 4.4 and Section 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims throughout: 'NC AutoML has been proven to be a valuable tool' (Section 5.1), 'NC AutoML can improve the AI product prototyping process' (Section 7). These claims rest on a single illustrative case study with no controlled comparison, inadequate for causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion claims 'This innovation is not confined to any specific sector; rather, it permeates all industries' (Section 7) based on one case study with one tool (Google Vertex AI), one use case (text classification), and one dataset. Section 5.4 acknowledges limitations but the framing in Sections 5.2 and 7 far exceeds the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for why the case study demonstrated positive results. For example, the success might be attributable to the simplicity of the text classification task, the quality of the specific dataset, or Google Vertex AI's particular strengths rather than NC AutoML in general."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures whether NC AutoML provides certain features (UI guidance, preview capability, performance metrics) and frames this as 'utility' and 'validity' of the framework for improving the prototyping process. No acknowledgment that the presence of features does not equal actual improvement in prototyping outcomes for non-experts."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Google Vertex AI' and 'Google AutoML' but provides no version information, API version, or date of access beyond 'Accessed: Jan. 01, 2024' in the reference."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. The NC AutoML approach uses a GUI-based workflow, not prompt-based interaction."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The only parameter mentioned is the 80% train / 20% validation-test split. No other hyperparameters are reported. The paper states 'AutoML required 5 hours to train' but provides no detail on what configurations or algorithms were used internally."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates a GUI-based AutoML tool."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We used a dataset with 3440 training examples' uploaded as CSV, with data previewed and class balance checked, but provides no detail on how the original data was collected, cleaned, or prepared for upload."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 'Limitations of Research' is a dedicated subsection listing four specific limitations of the study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 lists specific threats: the case study used only Google Vertex AI for classification, only one dataset and one ML functionality type, and identifies four concrete steps needed for generalizability (multiple providers, multiple use cases, user acceptance testing, multiple AI use cases)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 explicitly states what was NOT tested and what future research should address: multiple AutoML providers, multiple ML use cases, user acceptance testing with actual AI non-experts, and multiple AI use cases."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The customer support dataset of 3,440 examples used in the case study is not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states only 'We used a dataset with 3440 training examples' for customer request classification. No description of how the data was collected, from which source, over what time period, or what the data characteristics are beyond the number of examples."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The case study user is the author. The data source is customer support tickets, not a recruited sample."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of how the 3,440 customer support tickets were collected, filtered, labeled, or prepared before upload to Google AutoML. The paper goes directly from 'we used a dataset' to demonstrating the AutoML workflow."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'No explicit funding' under 'Funder information' and acknowledges employers (Adobe and Siemens) in the acknowledgments section."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Mario Truss (Adobe) and Marc Schmitt (Siemens). The acknowledgments thank 'their employer for allowing them to do independent research.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The authors' employers (Adobe and Siemens) do not make the specific product being evaluated (Google Vertex AI). Neither company has a direct financial stake in the paper's findings about NC AutoML for prototyping."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "The Declaration of Interest Statement reads: 'The authors declare that they have no known competing financial interests or personal relationships that could have appeared to influence the work reported in this article.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on any benchmark. It demonstrates a no-code AutoML tool for prototyping purposes."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model on any benchmark. The AutoML model trained in the case study uses the authors' own proprietary dataset."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The paper demonstrates an AutoML workflow, not model capability on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The case study was conducted by the authors themselves."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The case study uses the author as the NC AutoML user, not recruited participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no experimental conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No cost information is provided for using Google Vertex AI. The paper mentions '5 hours to train the ML model' but does not report any monetary cost, API costs, or compute resource details."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Only 'AutoML required 5 hours to train the ML model' is mentioned. No GPU/compute specifications, total API spend, or hardware details are provided."
       }
     }
   },
   "claims": [
     {
       "claim": "NC AutoML can be integrated into all three stages of the AI product prototyping process (ideation, prototyping, and testing).",
       "evidence": "Criteria-based evaluation in Section 4.4 (A1-A3) describes how Google AutoML supports ideation through documentation, prototyping through automated model training via UI, and testing through the preview function.",
       "supported": "weak"
     },
     {
       "claim": "NC AutoML provides insights into all four product success criteria: viability, feasibility, usability, and desirability.",
       "evidence": "Sections B1-B4 in Section 4.4 describe how AutoML metrics, confusion matrices, and preview UIs can inform each criterion. Based on features observed in the Google Vertex AI case study.",
       "supported": "weak"
     },
     {
       "claim": "NC AutoML solves or improves the four identified challenges in AI product prototyping (knowledge gaps, boundary objects, unpredictable AI, development complexity).",
       "evidence": "Sections C1-C4 in Section 4.4 argue that AutoML UI guidance bridges knowledge gaps (C1), acts as a boundary object (C2), creates transparency about AI capabilities (C3), and reduces development complexity (C4).",
       "supported": "weak"
     },
     {
       "claim": "AutoML can create comparable results to manual ML development.",
       "evidence": "Referenced from prior literature in Section 2.5: 'they provide proof that AutoML can create comparable results to manual ML' citing references [39], [40], [42], [46], [79]-[82]. No original evidence provided in this paper.",
       "supported": "moderate"
     },
     {
       "claim": "NC AutoML can reduce misinvestments in AI product ideas that are not feasible for productive development.",
       "evidence": "Section 5.1 argues this based on the ability to evaluate AI product ideas quickly with real data, but the claim rests on the single case study with no measurement of actual investment outcomes.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Self-referential evaluation design",
       "detail": "The evaluation criteria (A1-A3, B1-B4, C1-C4) were designed by the same authors who created the framework and conducted the case study. There is no independent validation or external evaluation by actual AI non-experts."
     },
     {
       "flag": "Claims significantly outrun evidence",
       "detail": "The conclusion states 'This innovation is not confined to any specific sector; rather, it permeates all industries' based on a single case study with one tool (Google Vertex AI), one dataset (3,440 customer support tickets), and one use case (text classification). The language of 'proven' (Section 5.1) is inappropriate for the evidence level."
     },
     {
       "flag": "No actual user study despite claims about non-experts",
       "detail": "The paper's central claim is about supporting AI non-experts, yet the case study was conducted by the authors themselves. The paper states 'The NC AutoML user had no prior knowledge of ML and no coding capabilities' but does not clarify whether this refers to one of the authors or an external participant, and no user acceptance testing was conducted."
     },
     {
       "flag": "Single case study generalization",
       "detail": "All framework validation relies on one case study with Google Vertex AI for text classification. While Section 5.4 acknowledges this limitation, the framework is nonetheless presented as validated and generalizable throughout Sections 5 and 7."
     },
     {
       "flag": "Conflict of interest not fully explored",
       "detail": "The first author works at Adobe, which offers AI platform products (Adobe Sensei is mentioned in Section 2.3). While they evaluate a Google product, the general finding that NC AutoML is valuable for prototyping serves the interests of all major AI platform providers including the authors' employer."
     }
   ],
   "cited_papers": [
     {
       "title": "AutoML: A Survey of the State-of-the-Art",
       "authors": ["X. He", "K. Zhao", "X. Chu"],
       "year": 2021,
       "doi": "10.1016/j.knosys.2020.106622",
       "relevance": "Comprehensive survey of AutoML techniques and capabilities, relevant to understanding automated ML development tools."
     },
     {
       "title": "PromptMaker: Prompt-based Prototyping with Large Language Models",
       "authors": ["E. Jiang"],
       "year": 2022,
       "doi": "10.1145/3491101.3503564",
       "relevance": "Explores using LLMs for AI product prototyping through prompts, directly relevant to LLM-based development tools."
     },
     {
       "title": "PromptChainer: Chaining Large Language Model Prompts through Visual Programming",
       "authors": ["T. Wu"],
       "year": 2022,
       "doi": "10.1145/3491101.3519729",
       "relevance": "Introduces LLM chaining for prototyping, relevant to agentic workflow composition and multi-step LLM pipelines."
     },
     {
       "title": "PromptInfuser: Bringing User Interface Mock-ups to Life with Large Language Models",
       "authors": ["S. Petridis", "M. Terry", "C. J. Cai"],
       "year": 2023,
       "doi": "10.1145/3544549.3585628",
       "relevance": "Integrates LLM responses into UI prototypes, relevant to AI product prototyping and LLM application development."
     },
     {
       "title": "ProtoAI: Model-Informed Prototyping for AI-Powered Interfaces",
       "authors": ["H. Subramonyam", "C. Seifert", "E. Adar"],
       "year": 2021,
       "doi": "10.1145/3397481.3450640",
       "relevance": "Proposes a framework for prototyping AI interfaces with functional models, directly relevant to AI product development."
     },
     {
       "title": "Vulnerabilities in AI Code Generators: Exploring Targeted Data Poisoning Attacks",
       "authors": ["D. Cotroneo", "C. Improta", "P. Liguori", "R. Natella"],
       "year": 2023,
       "doi": "10.48550/arXiv.2308.04451",
       "relevance": "Explores security vulnerabilities in AI code generation systems, relevant to AI safety and code quality concerns."
     },
     {
       "title": "Fits and Starts: Enterprise Use of AutoML and the Role of Humans in the Loop",
       "authors": ["A. Crisan", "B. Fiore-Gartland"],
       "year": 2021,
       "relevance": "Studies enterprise adoption of AutoML and human-in-the-loop dynamics, relevant to AI productivity tools and human-AI collaboration."
     },
     {
       "title": "Advances, challenges and opportunities in creating data for trustworthy AI",
       "authors": ["W. Liang"],
       "year": 2022,
       "doi": "10.1038/s42256-022-00516-1",
       "relevance": "Addresses data quality challenges for trustworthy AI development, relevant to AI safety and data integrity."
     },
     {
       "title": "Six Human-Centered Artificial Intelligence Grand Challenges",
       "authors": ["O. Ozmen Garibay"],
       "year": 2023,
       "doi": "10.1080/10447318.2022.2153320",
       "relevance": "Defines key challenges in human-centered AI, providing context for AI product development research."
     },
     {
       "title": "Democratizing artificial intelligence: How no-code AI can leverage machine learning operations",
       "authors": ["L. Sundberg", "J. Holmström"],
       "year": 2023,
       "doi": "10.1016/j.bushor.2023.04.003",
       "relevance": "Directly addresses AI democratization through no-code tools and MLOps, relevant to AI accessibility and tool evaluation."
     },
     {
       "title": "Accelerating Innovation With Generative AI: AI-Augmented Digital Prototyping and Innovation Methods",
       "authors": ["V. Bilgram", "F. Laarmann"],
       "year": 2023,
       "doi": "10.1109/EMR.2023.3272799",
       "relevance": "Explores generative AI for digital prototyping, relevant to LLM-based development tools and AI-augmented workflows."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The framework provides a conceptual guide for product managers wanting to use NC AutoML for prototyping, but no immediately usable tool or technique."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "The finding that no-code tools can help non-experts prototype AI products confirms widely-held expectations about low-code/no-code democratization."
     },
     "fear_safety": {
       "score": 0,
       "justification": "The paper briefly mentions ethical risks and black-box concerns but raises no novel safety or security concerns."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenges to existing claims or tools."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Google Vertex AI is publicly available and someone could follow along, but the paper provides no code, demo, or reproducible artifact."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from Adobe and Siemens, uses Google AutoML — known brands but not the most prominent AI labs."
     }
   }
 }