ai-research-survey

scan.json (25275B)

---

 {
   "paper": {
     "title": "Artificial Intelligence for Health Message Generation: Theory, Method, and an Empirical Study Using Prompt Engineering",
     "authors": ["Sue Lim", "Ralf Schmälzle"],
     "year": 2022,
     "venue": "Department of Communication, Michigan State University",
     "doi": ""
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided in the paper. The authors mention using Python, Google Colab, and various packages but do not release any code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The scraped tweets, AI-generated messages, and survey responses are not released. No dataset download link or data repository is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using Python, Google Colab, and packages like spacy, textacy, vader, sentence-transformers, snscrape, and the transformers package, but does not provide a requirements.txt, Dockerfile, or detailed version information for these dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the paper describes the general methodology (prompt engineering steps, computational analyses, survey design), it does not provide step-by-step reproduction instructions, scripts, or a README that would allow a researcher to replicate the experiments without significant guesswork."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report means and standard deviations but do not include confidence intervals or error bars. Figure 3 shows dot plots but no error bars or CI notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "T-tests are used to compare AI-generated vs. human-generated messages for reading ease, sentiment (Table 1), clarity, and quality (Table 2), with p-values reported (e.g., tclarity = 4.32, p < .01; tquality = 5.39, p < .01)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The power analysis specifies effect size d = .3. The results tables provide means and standard deviations with baseline context (e.g., AI clarity 3.77 vs. human 3.22 on a 1-5 scale), providing enough information to compute and understand the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports a power analysis: 'We also conducted a power analysis using the pwr package in R for a one-sample and one-sided t-test, with effect size d = .3 and significance level α = .05. Sample size of 100 was enough to detect significance at the power level of .9.' They also cite Kim & Cappella (2019) for sample sufficiency."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported alongside means in Tables 1 and 2 (e.g., Clarity: AI 3.77 (0.55), Human 3.22 (0.43); Flesch: AI 63.4 (17.9), Human 68.4 (20.3))."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study compares AI-generated messages to retweeted human-generated messages as a baseline, which serves as the comparison standard throughout both computational and human evaluation analyses."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The only comparison is against human-generated tweets. There is no comparison against other AI text generation systems (e.g., GPT-3 which was available at the time and mentioned in the related work). The prior study by Schmälzle and Wilcox (2022) using GPT-2 with fine-tuning is discussed but not directly compared."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses six different prompts but does not systematically ablate components (e.g., comparing prompting strategies, temperature settings, or model sizes). The prompt variations are described qualitatively but not evaluated in a controlled ablation framework."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The study uses multiple metrics: Flesch Reading Ease, VADER sentiment scores, n-gram analysis, topic modeling, semantic similarity (cosine similarity of sentence embeddings), and human ratings of clarity and quality."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "A human evaluation study with N=120 participants rated AI-generated and human-generated messages on quality and clarity using 5-point Likert scales (Tables 2 and Figure 3)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning benchmark study with train/test splits. The evaluation is a direct comparison of generated messages vs. human messages, so held-out test sets are not structurally applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by gender (male vs. female) in Figure 3, and by prompt type in the qualitative analysis. The semantic similarity analysis shows within-group and between-group comparisons."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure cases during prompt engineering: 'too many of the same messages without much informative content, too many names of official organizations or countries, or clearly false information.' Messages with exclusion criteria (false information, non-US references, unverifiable sources, recipes, non-English characters, repetitive phrases) are described."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that AI-generated messages were more similar to each other than human messages (msimilarity AI = .59 vs. msimilarity human = .43, p < 0.001), which is presented as a limitation of the AI approach. The non-significant results for reading ease and sentiment are also reported (Table 1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that: (1) AI-generated messages were on par with human ones in sentiment, reading ease, and semantic content — supported by Table 1 and Figure 2; (2) human evaluation showed AI messages ranked higher in quality and clarity — supported by Table 2 with significant t-test results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-adjacent claims such as 'the Bloom message engine has the potential to generate informative awareness messages that surpass the retweeted messages' and discusses prompt engineering as a way to 'strategically control qualities and tone.' However, the study design does not adequately control for confounds — e.g., AI messages were curated and cleaned while human messages were scraped from Twitter, introducing different production processes. The comparison is observational despite the causal framing."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "While limitations mention testing only one topic (folic acid) and one message type (awareness), the title and discussion make broad claims about 'AI for Health Message Generation' generally. The abstract does not bound claims to folic acid or the Bloom model. The discussion states 'we anticipate that this work will find many applications in health communication in the near future' without qualifying these claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for why AI messages were rated higher. For example, the AI messages were curated through a multi-step filtering process while human messages were simply the top retweeted tweets — the curation process itself (not AI quality) could explain the difference. The higher internal semantic similarity of AI messages is noted but not explored as a potential confound."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'Bloom 7B1' with '7 billion neural network parameters' and references Bigscience (2022). The specific model variant (bloom-7b1) is identified, and the Huggingface link is provided in the references."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "All six prompts are explicitly listed: 'It's National Folic Acid Awareness week', 'Every woman needs #folicacid every day,', 'Did you know, #Folicacid', 'Consuming folic acid is important because', 'Consuming folic acid:', and 'Folic acid during pregnancy'."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Hyperparameters are reported: maximum result length = 60 tokens, do_sample=True, temperature = 0.7, top_k=40, top_p=0.9. These are described in Step 3 of the Message Generation Protocol."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a straightforward prompt-to-generation pipeline with no tool use, retry logic, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data preprocessing pipeline is documented with counts: 42,646 raw tweets scraped, top 50 selected by retweet count, filtered for duplicates/non-English/promotional content yielding 28, plus 2 more for 30 total. AI message selection: 600 generated, 10 randomly selected per prompt (60 total), exclusion criteria applied, final 30 selected."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "There is a section titled 'Ethical Considerations and Limitations' that discusses multiple limitations spanning approximately two paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section discusses specific threats: testing only one topic (folic acid), focusing only on awareness messaging (not attitude/behavior change), generating only short messages, limited computational metrics, and the narrow participant sample (college students) with acknowledgment that it overlaps with but does not fully represent the target population."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope: 'this empirical research paper focuses only on the proximal goal of raising awareness' (footnote 1), tested only folic acid, only short social-media-style messages, and notes 'it will be necessary to expand these findings to other domains.' The footnotes repeatedly clarify what the paper does NOT claim."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data (scraped tweets, generated messages, survey responses) are not available for independent verification. No data repository or supplementary data files are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail: tweets scraped using snscrape with hashtags #folicacid and #folate from Twitter (no date constraints), yielding 42,646 raw messages. AI messages generated via Bloom with specified parameters. Survey conducted via Qualtrics with course credit compensation."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Participants were recruited from a study pool and received course credits as compensation for the study.' The sample is characterized: N=120, 70% female (n=84), college students. The connection to the target population (potential parents, women of childbearing age) is discussed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented across Steps 1-4 of the Method section, with counts at each stage: 42,646 tweets → top 50 → 28 after filtering → 30 final human messages. For AI: 600 generated → 60 randomly selected → exclusion criteria applied → 30 final. Survey: responses from unrealistically fast completions (<4 min) or incomplete surveys discarded, yielding N=120."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. There is no mention of grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: 'Department of Communication, Michigan State University, East Lansing, USA' for both authors. Since they are evaluating an open-source model (Bloom) and not their own product, there is no inherent vendor conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of funding disclosure does not confirm unfunded status."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes Bloom was trained on '1.5 TB of pre-processed text from 45 natural and 12 programming languages' but does not state the training data cutoff date. This is relevant because the folic acid tweets used for comparison could have been in Bloom's training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the retweeted folic acid tweets or similar health messages could have appeared in Bloom's training data. Since Bloom was trained on internet text, the comparison tweets may have been in the training corpus, which would affect the validity of the comparison."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not address whether the health messages or related folic acid content were part of Bloom's training data. Since Bloom was trained on Wikipedia and other web text, folic acid information was almost certainly in the training data, which could explain the model's ability to generate accurate health content."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration link (OSF, AsPredicted, or similar) is mentioned. The study involves N=120 human participants evaluating messages."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the study 'was approved by the local review board' in the Participants section."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Basic demographics are reported: N=120, 70% female (n=84), recruited from a university study pool (college students). Gender breakdown of results is shown in Figure 3."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Exclusion criteria are stated: 'Responses from participants who completed the survey at an unrealistically fast speed (<4 minutes) or failed to complete the survey were discarded.'"
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Randomization is described: 'Message order was randomized within each block and approximately half of the sample started with the first question, the other half started with the second block.'"
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'participants were not told in advance which of the messages were AI-generated or human generated; they were only told that the purpose of the study was to evaluate health messages related to folic acid.'"
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions that fast or incomplete responses were discarded to yield N=120, but does not report how many participants started the study, how many were excluded, or specific dropout numbers. The initial N before exclusions is not stated."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports practical time costs: 'Loading the Bloom 7B1 model into Google Colab and generating 600 messages took a little over an hour.' The feasibility section discusses time requirements for the overall pipeline."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Google Colab is mentioned as the platform and generation time is reported (~1 hour for 600 messages), the specific GPU type, GPU hours, or total computational budget is not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "The Bloom message engine is easy to use and can generate messages via prompting without fine-tuning.",
       "evidence": "RQ1 results in the Feasibility section: the system required only Python coding with available online resources, and generating 600 messages took about an hour on Google Colab.",
       "supported": "moderate"
     },
     {
       "claim": "AI-generated messages are on par with human-generated messages in terms of sentiment, reading ease, and semantic content.",
       "evidence": "Table 1: No significant differences in Flesch Reading Ease (AI 63.4 vs. Human 68.4, p=.32) or VADER sentiment (AI .25 vs. Human .23, p=.87). Semantic similarity analysis and topic modeling in Figure 2.",
       "supported": "moderate"
     },
     {
       "claim": "AI-generated messages were rated significantly higher than human-generated messages in clarity and quality by human evaluators.",
       "evidence": "Table 2: t-test results for clarity (t=4.32, p<.01) and quality (t=5.39, p<.01) with N=120 participants. AI clarity 3.77 vs. human 3.22; AI quality 3.65 vs. human 3.12.",
       "supported": "moderate"
     },
     {
       "claim": "Prompt engineering can be used to strategically control the qualities and tone of AI-generated messages.",
       "evidence": "Qualitative observations in the RQ2 results: prompts with #folicacid generated tweet-like content with hashtags, while prompts without hashtags generated more formal, factual content.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "Using the Bloom 7B1 language model with prompt engineering, AI-generated folic acid awareness messages were comparable to retweeted human messages on computational metrics (reading ease, sentiment, semantic similarity) and were rated significantly higher by 120 human evaluators on clarity and quality. The system generated 600 messages in about an hour without requiring model fine-tuning. AI-generated messages showed higher internal similarity than human messages, suggesting less diversity in AI output.",
   "red_flags": [
     {
       "flag": "Unfair comparison standard",
       "detail": "AI-generated messages underwent a multi-step curation process (generating 600, randomly selecting 60, applying exclusion criteria, cleaning) while human messages were simply the top 30 retweeted tweets with basic filtering. The AI messages were effectively cherry-picked from a larger pool while human messages were not similarly curated. This asymmetric curation could explain the quality difference rather than AI superiority."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Bloom was trained on Wikipedia, Semantic Scholar, and other web text which almost certainly included folic acid health information. The model's ability to generate accurate health messages may reflect memorized training data rather than genuine generation capability, but this is never discussed."
     },
     {
       "flag": "Small and narrow message sample",
       "detail": "Only 30 AI messages and 30 human messages were compared. With such a small sample of messages (not participants), individual message quality could drive the results. The single health topic (folic acid) limits generalizability despite broad claims about 'AI for Health Message Generation.'"
     },
     {
       "flag": "No alternative explanations for quality difference",
       "detail": "The paper does not consider that the curation process, the formal/informative style of Bloom outputs vs. casual Twitter style, or the removal of false/problematic content from AI messages could explain the higher human ratings, rather than intrinsic AI quality."
     }
   ],
   "cited_papers": [
     {
       "title": "Pre-train, prompt, and predict: A systematic survey of prompting methods in natural language processing",
       "authors": ["P. Liu", "W. Yuan", "J. Fu", "Z. Jiang", "H. Hayashi", "G. Neubig"],
       "year": 2021,
       "relevance": "Systematic survey of prompting methods for LLMs, directly relevant to understanding prompt engineering techniques used in AI systems."
     },
     {
       "title": "All the news that's fit to fabricate: AI-generated text as a tool of media misinformation",
       "authors": ["S. E. Kreps", "M. McCain", "M. Brundage"],
       "year": 2020,
       "relevance": "Examines AI-generated text for misinformation, relevant to safety and adversarial use of LLM text generation."
     },
     {
       "title": "Release strategies and the social impacts of language models",
       "authors": ["I. Solaiman", "M. Brundage", "J. Clark"],
       "year": 2019,
       "relevance": "Discusses social impacts and release strategies for language models, relevant to AI safety and responsible deployment."
     },
     {
       "title": "CTRL: A conditional transformer language model for controllable generation",
       "authors": ["N. S. Keskar", "B. McCann", "L. R. Varshney", "C. Xiong", "R. Socher"],
       "year": 2019,
       "relevance": "Presents controllable text generation with transformer models, relevant to understanding AI code/text generation capabilities."
     },
     {
       "title": "Transformers: State-of-the-Art natural language processing",
       "authors": ["T. Wolf", "L. Debut", "V. Sanh"],
       "year": 2020,
       "relevance": "The HuggingFace Transformers library used as infrastructure for LLM-based research and applications."
     },
     {
       "title": "Climbing towards NLU: On meaning, form, and understanding in the age of data",
       "authors": ["E. M. Bender", "A. Koller"],
       "year": 2020,
       "relevance": "Critical examination of what LLMs actually learn about language, relevant to understanding LLM capabilities and limitations."
     },
     {
       "title": "The curious case of neural text degeneration",
       "authors": ["A. Holtzman", "J. Buys", "L. Du", "M. Forbes", "Y. Choi"],
       "year": 2019,
       "relevance": "Addresses text degeneration in neural language models and proposes nucleus sampling, relevant to LLM output quality."
     },
     {
       "title": "Behavioral use licensing for responsible AI",
       "authors": ["D. Contractor", "D. McDuff", "J. Haines"],
       "year": 2020,
       "relevance": "Proposes responsible AI licensing framework (RAIL), relevant to AI safety and governance of LLM deployment."
     }
   ]
 }