ai-research-survey

scan.json (28037B)

---

 {
   "paper": {
     "title": "Prompt Sapper: A LLM-Empowered Production Tool for Building AI Chains",
     "authors": [
       "Yu Cheng",
       "Jieshan Chen",
       "Qing Huang",
       "Zhenchang Xing",
       "Xiwei Xu",
       "Qinghua Lu"
     ],
     "year": 2023,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2306.12028",
     "doi": "10.1145/3638247"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": ["case-study", "qualitative"],
   "key_findings": "Prompt Sapper, a no-code visual programming IDE for building AI chains on foundation models, significantly reduces task completion time compared to Python/PyCharm (mean 1,689s vs 2,366s, p=0.0004) while maintaining equivalent correctness in a within-subject study with 18 participants. A second user study with 12 participants showed that LLM-based co-pilots for requirement elicitation and AI chain skeleton generation received favorable ratings (mean ≥3.5/5 across all dimensions), with 18/24 generated skeletons directly executable without modification.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub URL (https://github.com/YuCheng1106/PromptSapper) and states 'we released the source code of our tool at our GitHub repository.' However, the sapperchain Python library required to execute downloaded AI chain code is explicitly noted as 'not open sourced' (Section 4.3.1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The implementation of Sapper IDE and all experiment data and results can be downloaded at our Github repository' (Section 5 footnote and Section 8)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications (requirements.txt, Dockerfile, dependency lists with versions) are mentioned in the paper. The tool is described as web-based but no deployment or setup instructions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are described in the paper. The study procedure is described at a high level but there are no specific instructions for reproducing the user studies or running the tool."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports means with standard deviations in parentheses for all conditions and tasks, e.g., 'Mean=1,689.00s, Std=447.49s' (Section 5.2.1). Figure 3 shows usability scores as bar charts."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Paired t-tests are used throughout with Benjamini-Hochberg correction for multiple comparisons. Specific p-values reported: time difference p=0.0004, diffuseness p=0.0151, visibility p=0.0006 (Section 5.2)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "While formal effect sizes (Cohen's d) are not computed, the paper provides means and standard deviations for both conditions allowing effect size assessment: Sapper V2 mean 1,689s vs Python mean 2,366s with t-statistic of 4.54 (Section 5.2.1)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 18 participants were recruited for study 1 or 12 for study 2. No power analysis is reported. The sample sizes appear chosen by convenience."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported alongside means in Table 1 for all conditions and tasks, e.g., 'Mean=1,689.00s, Std=447.49s' and '475.27 (228.58)' format."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study compares Sapper V2 against Python/PyCharm (native programming baseline) and Sapper V1 (block view only, ablation baseline) in a within-subject design (Section 5.1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Python is the most common language for LLM development and PyCharm is a widely-used contemporary IDE. These are appropriate and current baselines for the task of building AI chains."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Sapper V1 (Block View only) vs Sapper V2 (Block View + Design View) serves as an ablation study isolating the contribution of the Design View component (Section 5.2.2)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The study measures task completion time, correctness (number of correctly completed tasks), and 9 cognitive dimension usability scores (Section 5.2). The co-pilot study adds 5 further rating dimensions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The entire evaluation methodology consists of user studies with human participants (18 in study 1, 12 in study 2) who directly use and evaluate the tools (Sections 5 and 6)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a user study evaluating a tool through human performance, not a machine learning benchmark evaluation. The concept of held-out test sets does not structurally apply."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down time spent per task (Tasks 1-4). Figure 3 shows 9 individual cognitive dimension scores. Figures 4 and 5 break down co-pilot ratings across 5 and 3 dimensions respectively."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2.2 discusses specific issues: Design View doesn't support copy-paste, doesn't generate if-else/while loops, task decomposition co-pilot requires significant response time. Section 6.2.2 notes 6/24 skeletons required modifications."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that Sapper V2 showed no significant improvement over V1 in time (p=0.62, Section 5.2.2). It also reports Task 2 took slightly longer in V2 than V1 (Table 1: 529.94s vs 485.72s). Several Design View limitations are explicitly discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the tool 'evaluated and demonstrated the efficiency and correctness.' Efficiency is supported by significant time savings (p=0.0004). Correctness is supported by equivalent correctness to Python (no significant difference, p=0.59). The abstract also claims co-pilots 'enhance the efficiency and effectiveness,' supported by the second user study's favorable ratings."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims Sapper 'can significantly save time' compared to Python. The within-subject design with counterbalanced order across 6 possible tool orderings (Section 5.1.1) provides adequate causal identification for tool-level comparisons."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Prompt Sapper can 'democratize the usage of LLMs, making AI accessible to a wider range of individuals, including non-technical people' (Section 1), but all 30 participants were 18-25 year old university students majoring in CS, SE, AI, or Big Data. The generalization to non-technical users is unsupported."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses counterbalanced order to control for sequence effects but does not discuss alternative explanations such as novelty effects, experimenter demand characteristics, task designs that may favor visual programming, or whether the observed time differences stem from API lookup difficulty rather than tool quality."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures task completion time and Likert-scale usability scores but frames results as demonstrating 'practicality,' 'low entry barrier,' and 'democratization.' The gap between these measured proxies and the broader claimed outcomes (especially for non-technical users) is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'gpt-3.5-turbo,' 'text-davinci-003,' and 'DALL-E' as pre-installed engines (Section 4.3.3) but does not specify exact version snapshots or dates. 'gpt-3.5-turbo' changes over time without a version specifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The co-pilots are described functionally (e.g., 'prompted as an infinite questioner,' 'converts the high-level intention into the main steps') but the actual system prompts used by the requirement elicitation and skeleton generation co-pilots are not provided anywhere in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.3.3 mentions configurable parameters (temperature, maximum length, Top P, frequency penalty, presence penalty) in Engine Management but does not state what values were used during the user studies."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main contribution is describing the Prompt Sapper tool architecture in detail across Sections 3-4, including Design View workflow (Fig. 1), Block View visual programming (Fig. 2), co-pilot interaction cycles, worker composition and cooperation patterns, and how conversation histories are transmitted to the LLM."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The study procedure is described (counterbalanced order, screen recording, questionnaires) but how the raw data (screen recordings, time measurements, correctness assessments) was processed into the reported statistics is not documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 (Discussion) discusses future enhancements and challenges but is forward-looking rather than a limitations section. There is no dedicated 'Limitations' or 'Threats to Validity' section addressing the current study's weaknesses."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The Discussion section covers future work directions (enhanced co-pilots, testing challenges, deployment) but does not address specific methodological threats like the student-only sample, small N, or potential demand effects."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its claims about 'democratization' or 'non-technical users' to the actually tested population of CS/SE/AI students."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'all experiment data and results can be downloaded at our Github repository' (Section 8, with URL https://github.com/YuCheng1106/PromptSapper)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.1 describes the procedure: within-subject design, counterbalanced order, training session, warm-up tasks, three task sets with similar difficulty, screen recording, and post-task questionnaires. Section 6.1 describes the co-pilot study procedure."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Study 1 says only 'We recruited 18 participants' with no description of how. Study 2 says 'recruited another 12 participants... through public channels' which is vague. No description of recruitment channels, response rates, or potential selection bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how screen recordings were analyzed, how correctness was determined from task outputs, or how time measurements were extracted. The pipeline from raw data collection to reported statistics is not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This work is partially supported by the National Nature Science Foundation of China under Grant (62262031).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Jiangxi Normal University (China) and CSIRO's Data61 (Australia). These are the developers of the evaluated tool."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The National Natural Science Foundation of China is a government funding agency with no financial stake in whether Prompt Sapper outperforms Python."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The authors developed and evaluate their own tool, but this potential conflict is not explicitly acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper evaluates a visual programming tool through user studies, not a pre-trained model's capability on benchmarks. Contamination criteria do not apply."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of pre-trained model capability is performed. The evaluation is a user study comparing programming tools."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of pre-trained model capability is performed. Contamination does not structurally apply."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration (OSF, AsPredicted, or any registry) is found in the paper."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned. The only consent reference is 'We obtained their approval for this' regarding screen recording (Section 5.1.1 footnote), which is participant consent, not ethics board review."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Study 1 reports age range (18-25), majors (CS, SE, AI), programming experience (0-1, 1-3, 3+ years), gender (10M, 8F), and prior tool experience. Study 2 reports age (18-25), specializations (AI, CS, Big Data), and experience levels (Section 5.1.1 and 6.1)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "Participant characteristics are described (e.g., 'None of them have used visual programming tools before, and all have learned Python and use PyCharm before') but these are stated as sample observations, not as pre-defined inclusion/exclusion criteria with a screening process."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.1 describes counterbalanced order: 'since we had three different tools to choose from, there were a total of six possible combinations of orders in which the tools could be used. Each combination of orders was used by three participants.'"
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "In a within-subject study comparing visually distinct programming tools (PyCharm vs Sapper V1 vs Sapper V2), participant blinding is structurally not feasible. Outcome measures (time, correctness via test cases, self-reported usability) are either automated or self-reported."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report whether all recruited participants completed all tasks. It starts with 18 (study 1) and 12 (study 2) and reports results for these numbers, but does not explicitly state whether anyone dropped out or was excluded."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The tool calls LLM APIs (GPT-3.5-turbo, text-davinci-003) for co-pilots and worker execution, but no API costs, token counts, or latency measurements are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget, hardware specifications, or total API spend for the user studies or tool development is reported."
       }
     }
   },
   "claims": [
     {
       "claim": "Sapper V2 significantly reduces task completion time compared to Python/PyCharm",
       "evidence": "Paired t-test: t=4.54, p=0.0004. Sapper V2 mean 1,689s (Std=447.49s) vs Python mean 2,366s (Std=536.70s). Table 1 shows per-task breakdown (Section 5.2.1).",
       "supported": "strong"
     },
     {
       "claim": "Sapper V2 achieves equivalent correctness to Python",
       "evidence": "No significant difference in correctness: t=-0.53, p=0.59 (Section 5.2.1).",
       "supported": "moderate"
     },
     {
       "claim": "Sapper V2 is significantly more concise (diffuseness) and modifiable (visibility) than Python",
       "evidence": "Diffuseness: p=0.0151; Visibility: p=0.0006. Other 7 cognitive dimensions showed no significant difference (Section 5.2.1, Figure 3).",
       "supported": "strong"
     },
     {
       "claim": "Design View (V2 vs V1) does not significantly improve time or usability",
       "evidence": "No significant difference between V1 (Mean=1,751s) and V2 (Mean=1,689s) in time (p=0.62). No significant usability differences across 9 dimensions (Section 5.2.2).",
       "supported": "moderate"
     },
     {
       "claim": "The requirement elicitation co-pilot is useful and easy to use",
       "evidence": "Mean ratings ≥3.5/5 across all five dimensions. Over 70% of sessions rated 4-5 on usefulness and diversity; over 80% rated 4-5 on ease-of-use and relevance (Section 6.2.1, Figure 4).",
       "supported": "moderate"
     },
     {
       "claim": "The AI chain skeleton generation co-pilot produces usable skeletons",
       "evidence": "18/24 generated skeletons were directly executable; remaining 6 needed only minor prompt modifications. All participants said skeletons facilitated development (Section 6.2.2, Figure 5).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Small, non-representative sample",
       "detail": "All 30 participants across both studies were 18-25 year old university students majoring in CS, SE, AI, or Big Data. The paper generalizes to 'non-technical people' and claims to 'democratize the usage of LLMs' based on this narrow sample of technically trained students."
     },
     {
       "flag": "Authors evaluating own tool",
       "detail": "The authors developed Prompt Sapper and conducted all evaluations themselves. No independent evaluation or external evaluators were involved. This self-comparison bias is not acknowledged anywhere in the paper."
     },
     {
       "flag": "Incomplete reproducibility",
       "detail": "The sapperchain Python library required to execute downloaded AI chain code is explicitly described as 'not open sourced' (Section 4.3.1), meaning the tool cannot be fully reproduced or independently tested despite the source code release claim."
     },
     {
       "flag": "No ethics approval for human subjects research",
       "detail": "Two user studies involving 30 human participants were conducted with no mention of IRB or ethics board approval, only participant consent for screen recording."
     },
     {
       "flag": "Claims exceed evidence",
       "detail": "The paper makes broad claims about 'democratization,' 'production tool,' and benefiting 'non-technical people' (abstract, Section 1), but the user study tasks were simple programming constructs (if-else, while loops, variables) tested with CS/SE/AI students. No evidence supports claims about production use or non-technical users."
     }
   ],
   "cited_papers": [
     {
       "title": "Ai chains: Transparent and controllable human-ai interaction by chaining large language model prompts",
       "authors": ["Tongshuang Wu", "Michael Terry", "Carrie Jun Cai"],
       "year": 2022,
       "relevance": "Foundational work on chaining LLM prompts for transparency and controllability, directly motivating the AI chain methodology."
     },
     {
       "title": "Promptchainer: Chaining large language model prompts through visual programming",
       "authors": ["Tongshuang Wu", "Ellen Jiang", "Aaron Donsbach", "Jeff Gray", "Alejandra Molina", "Michael Terry", "Carrie J Cai"],
       "year": 2022,
       "relevance": "Visual programming approach for chaining LLM prompts, a direct precursor to Prompt Sapper's block-based approach."
     },
     {
       "title": "PromptMaker: Prompt-based Prototyping with Large Language Models",
       "authors": ["Ellen Jiang", "Kristen Olson", "Edwin Toh", "Alejandra Molina", "Aaron Donsbach", "Michael Terry", "Carrie J Cai"],
       "year": 2022,
       "relevance": "Prompt-based prototyping tool for ML functionality, demonstrating the need for tool support in prompt engineering."
     },
     {
       "title": "PCR-Chain: Partial Code Reuse Assisted by Hierarchical Chaining of Prompts on Frozen Copilot",
       "authors": ["Qing Huang", "Jiahui Zhu", "Zhilong Li", "Zhenchang Xing", "Changjing Wang", "Xiwei Xu"],
       "year": 2023,
       "relevance": "Hierarchical prompt chaining for code reuse tasks, demonstrating AI chain methodology applied to software engineering."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "brian ichter", "Fei Xia", "Ed Chi", "Quoc V Le", "Denny Zhou"],
       "year": 2022,
       "relevance": "Foundational chain-of-thought prompting technique that underpins multi-step reasoning in LLM-based AI chains."
     },
     {
       "title": "Reflexion: an autonomous agent with dynamic memory and self-reflection",
       "authors": ["Noah Shinn", "Beck Labash", "Ashwin Gopinath"],
       "year": 2023,
       "arxiv_id": "2303.11366",
       "relevance": "Autonomous LLM agent with self-reflection capability, representing the agentic AI paradigm that AI chains aim to systematize."
     },
     {
       "title": "Large Language Models Are Human-Level Prompt Engineers",
       "authors": ["Yongchao Zhou", "Andrei Ioan Muresanu", "Ziwen Han", "Keiran Paster", "Silviu Pitis", "Harris Chan", "Jimmy Ba"],
       "year": 2023,
       "arxiv_id": "2211.01910",
       "relevance": "Demonstrates LLMs can generate effective prompts automatically, supporting the 'magic enhancing magic' concept in Prompt Sapper."
     },
     {
       "title": "Large Language Models are Zero-Shot Reasoners",
       "authors": ["Takeshi Kojima", "Shixiang Gu", "Machel Reid", "Yutaka Matsuo", "Yusuke Iwasawa"],
       "year": 2022,
       "relevance": "Zero-shot reasoning via 'let's think step by step' prompting, a key technique for LLM task decomposition."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans", "Quoc Le", "Ed Chi", "Denny Zhou"],
       "year": 2022,
       "arxiv_id": "2203.11171",
       "relevance": "Self-consistency as a strategy to improve LLM reasoning, relevant to prompt engineering methodology."
     },
     {
       "title": "On the opportunities and risks of foundation models",
       "authors": ["Rishi Bommasani", "Drew A Hudson", "Ehsan Adeli"],
       "year": 2021,
       "arxiv_id": "2108.07258",
       "relevance": "Comprehensive analysis of foundation model opportunities and risks that frames the context for AI chain development."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper establishing in-context few-shot learning, the foundation for prompt-based AI services."
     },
     {
       "title": "Standing on the shoulders of giant frozen language models",
       "authors": ["Yoav Levine", "Itay Dalmedigos", "Ori Ram"],
       "year": 2022,
       "relevance": "Proposed using multiple calls on frozen LLMs to complete complex tasks, directly influencing AI chain decomposition methodology."
     },
     {
       "title": "Progprompt: Generating situated robot task plans using large language models",
       "authors": ["Ishika Singh", "Valts Blukis", "Arsalan Mousavian"],
       "year": 2022,
       "arxiv_id": "2209.11302",
       "relevance": "Code-like prompts for robot task planning with LLMs, demonstrating semi-structured prompt patterns used in AI chains."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "A no-code IDE for building AI chains is practically useful, but the core sapperchain library is not open-sourced, limiting immediate adoption."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Results confirm expected findings that visual tools are easier than text-based coding for non-experts."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the paper focuses on tool usability."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward tool presentation and evaluation."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Source code on GitHub and a web prototype at promptsapper.tech, but the required sapperchain library is closed-source."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "CSIRO's Data61 is a reputable Australian research organization and the paper is published in ACM TOSEM, but neither are household names."
     }
   }
 }