ai-research-survey

scan.json (22137B)

---

 {
   "paper": {
     "title": "Can AI Serve as a Substitute for Human Subjects in Software Engineering Research?",
     "authors": [
       "Marco Gerosa",
       "Bianca Trinkenreich",
       "Igor Steinmacher",
       "Anita Sarma"
     ],
     "year": 2023,
     "venue": "arXiv preprint",
     "arxiv_id": "2311.11081"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or repository is released. The paper provides links to shared ChatGPT conversations (chat.openai.com/share/...) as examples, but no analysis code or scripts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset is released. The paper includes illustrative ChatGPT conversation links and a comparison table (Table 1) with survey data, but the underlying data from the referenced survey [14] is not released as part of this paper."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a vision/position paper with no computational experiments requiring environment specification."
       },
       "reproduction_instructions": {
         "applies": false,
         "answer": false,
         "justification": "This is a vision paper proposing conceptual approaches. There are no experiments to reproduce beyond the illustrative ChatGPT conversations, for which shared links are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a vision paper. The only quantitative comparison (Table 1) shows absolute differences between LLM-generated and real survey responses, but no statistical analysis is performed on these."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No statistical experiments are conducted. The Table 1 comparison is illustrative, not a rigorous statistical evaluation."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No statistical experiments are conducted in this vision paper."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No empirical study with a sample is conducted. The paper is a conceptual/vision paper."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs are conducted. This is a vision paper with illustrative examples."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "This is a vision paper that does not propose a system or method to be evaluated against baselines. The Table 1 comparison is illustrative, not a formal evaluation."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No formal evaluation is conducted, so baselines are not applicable."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate. This is a conceptual paper."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No formal evaluation is conducted in this vision paper."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs are produced that require human evaluation. The paper is a conceptual discussion."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with training/test splits are conducted."
       },
       "per_category_breakdown": {
         "applies": false,
         "answer": false,
         "justification": "No formal evaluation is conducted that would require per-category breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 (Open Problems and Research Opportunities) extensively discusses limitations and failure modes: ethical considerations, bias in AI-generated data, feedback loops in AI training, complexity in persona differentiation, and challenges with demographic representation."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses negative aspects and limitations throughout: Section 3.1 on bias and fairness concerns, Section 3.2 on validity challenges, and the related work section cites Dominguez-Olmedo et al. [34] and Lee et al. [35] who found LLMs fail to represent certain subpopulations."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract makes appropriately hedged claims: 'we explore the potential,' 'could offer scalable and efficient means,' 'AI could augment aspects of data gathering.' These vision-level claims are supported by the conceptual discussion and illustrative examples in the paper."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It is a vision/position paper discussing potential future applications of LLMs in qualitative research."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is appropriately scoped to software engineering research and explicitly states it is a 'vision paper' exploring conceptual possibilities. The conclusion explicitly states: 'This paper does not propose to replace human subjects, but rather to explore the boundaries of AI's capabilities.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses multiple alternative perspectives: Section 4 (Related Work) cites work both supporting and questioning LLM fidelity as human proxies (e.g., Dominguez-Olmedo et al. questioning LLM survey responses, Lee et al. showing LLMs fail for some subpopulations). Section 3 discusses bias, feedback loops, and the risk that LLM outputs may reflect training data rather than genuine human perspectives."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states the illustrative examples were 'generated using GPT-4' but provides no version or snapshot date (e.g., gpt-4-0314 vs gpt-4-0613). The acknowledgments mention 'ChatGPT v4 was used for copy editing.'"
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The actual prompts used for the illustrative examples are provided in the appendices (Appendix A for focus groups, Appendix B for surveys), and shared ChatGPT conversation links are provided for each example (Figures 1-4)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, etc.) are reported for the GPT-4 interactions. Section 3.2 discusses 'setting the right level of randomness' as an open problem but does not report what settings were used for the illustrative examples."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The illustrative examples are simple prompt-response interactions with ChatGPT."
       },
       "data_preprocessing_documented": {
         "applies": false,
         "answer": false,
         "justification": "This is a vision paper with no data preprocessing pipeline. The illustrative examples use raw ChatGPT outputs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 (Open Problems and Research Opportunities) serves as an extensive limitations discussion, covering ethical considerations, bias, validity concerns, feedback loops, and persona differentiation challenges across 4 pages."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 discusses specific threats: embedded biases and stereotypes in LLM training data (Section 3.1), AI 'echo chambers' from feedback loops (Section 3.2), the caveat that prior research may be in training data invalidating replication studies (Section 3.2), and difficulty of precise demographic representation (Section 3.3)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: it is a 'vision paper' (stated multiple times), 'we neither believe nor desire for AI to completely replace human subjects,' and it is scoped to 'qualitative data collection in software engineering research.' The conclusion clarifies: 'This paper does not propose to replace human subjects.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw survey data from [14] used for the Table 1 comparison is not made available. The ChatGPT conversation links may expire and are not archived."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "For the illustrative examples, the data collection is described: prompts were given to GPT-4 via ChatGPT, and the survey comparison uses demographic data from a prior study [14] of 242 OSS contributors. The prompts and contexts are shown in the paper and appendices."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are recruited in this study. The paper discusses using AI as a substitute for human participants conceptually."
       },
       "data_pipeline_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data pipeline exists. The illustrative examples are direct ChatGPT interactions with no transformation steps."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Acknowledgments) states: 'The National Science Foundation partially supports this work under grant numbers 2236198, 2235601, 2247929, 2303043, and 2303042.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Northern Arizona University (Gerosa, Steinmacher) and Oregon State University (Trinkenreich, Sarma). No conflict with evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The National Science Foundation is an independent government funding agency with no financial stake in whether LLMs can substitute for human subjects in SE research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. It uses GPT-4 for illustrative examples of persona-based prompting, not for benchmark evaluation."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted. However, the paper does acknowledge the contamination concern in Section 3.2: 'the caveat that prior research could have been incorporated into the models' training datasets.'"
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted in this vision paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a vision paper with no method whose cost would be meaningful to report."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a vision paper with no significant computation performed."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM-generated survey responses exhibit an average deviation of just 4% from actual survey data across five questions about OSS contribution motivations.",
       "evidence": "Table 1 shows absolute differences between real survey data from [14] (242 OSS contributors) and GPT-4 generated responses across 5 Likert-scale questions with 6 response options. Deviations range from 1% to 14% per response option.",
       "supported": "weak"
     },
     {
       "claim": "Persona-based prompting can generate responses that are coherent with findings from previous research on gender differences in OSS contributions.",
       "evidence": "Section 2.1 compares two ChatGPT persona interviews (Figures 1-2): a male persona emphasized 'technical hurdles and complex problem-solving' while a female persona emphasized 'community collaboration, learning experiences.' Authors cite consistency with their prior work [14] showing men are more motivated by fun/technical challenges and women by reciprocity/kinship.",
       "supported": "weak"
     },
     {
       "claim": "AI could augment aspects of data gathering in software engineering research but cannot replace the nuanced, empathetic understanding inherent in human subjects.",
       "evidence": "This is the paper's central thesis, discussed throughout and in the conclusion (Section 5). It is a position statement supported by the conceptual discussion in Sections 2-3 and related work in Section 4.",
       "supported": "moderate"
     },
     {
       "claim": "Multi-persona prompting can simulate focus group dynamics with distinct voices building upon each other's contributions.",
       "evidence": "Section 2.2 and Appendix A present a 4-round focus group simulation generated by GPT-4 with 5 personas designing a web portal. The conversation shows personas referencing and building on each other's suggestions.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "theoretical",
     "qualitative"
   ],
   "key_findings": "This vision paper proposes using LLMs as substitutes for human subjects in qualitative SE research through three prompting strategies: persona-based prompting for interviews, multi-persona dialogue for focus groups, and mega-persona responses for surveys. An illustrative comparison of GPT-4 survey responses with actual data from 242 OSS contributors showed an average 4% deviation across 5 questions. The paper identifies numerous open problems including bias propagation, feedback loops in AI training, demographic representation precision, and ethical concerns, concluding that an integrated approach combining AI and human data will yield the best outcomes.",
   "red_flags": [
     {
       "flag": "Illustrative examples presented as evidence",
       "detail": "The paper presents a handful of ChatGPT conversation examples and a single 5-question survey comparison as evidence for the viability of AI-generated qualitative data, but acknowledges this is not an 'exhaustive validation.' The 4% average deviation claim is based on only 5 questions from a single survey with no statistical testing of the difference."
     },
     {
       "flag": "Confirmation bias in example selection",
       "detail": "The persona-based interview examples (Figures 1-2) are selected to show consistency with the authors' own prior research [14] on gender differences in OSS. No examples of cases where the LLM produced responses inconsistent with known findings are shown."
     },
     {
       "flag": "Training data contamination in validation",
       "detail": "The survey comparison uses data from the authors' own 2021 ICSE paper [14], which GPT-4 may have seen during training. The paper acknowledges this concern in Section 3.2 ('the caveat that prior research could have been incorporated into the models' training datasets') but still presents the comparison as supportive evidence."
     },
     {
       "flag": "No rigorous empirical validation",
       "detail": "As a self-described vision paper, the work provides no controlled experiments, statistical tests, or systematic evaluation of the proposed approaches. The claims about LLM fidelity rest entirely on illustrative examples and references to other researchers' findings."
     }
   ],
   "cited_papers": [
     {
       "title": "Out of one, many: Using language models to simulate human samples",
       "authors": ["Lisa P. Argyle", "Ethan C. Busby", "Nancy Fulda", "Joshua R. Gubler", "Christopher Rytting", "David Wingate"],
       "year": 2023,
       "relevance": "Directly relevant to using LLMs as proxies for human populations in research, introducing the concept of 'algorithmic fidelity.'"
     },
     {
       "title": "Can AI language models replace human participants?",
       "authors": ["Danica Dillion", "Niket Tandon", "Yuling Gu", "Kurt Gray"],
       "year": 2023,
       "doi": "10.1016/j.tics.2023.04.008",
       "relevance": "Discusses conditions under which LLMs might replace human participants in psychological science research."
     },
     {
       "title": "Using large language models in psychology",
       "authors": ["Dorottya Demszky", "Diyi Yang", "David S. Yeager"],
       "year": 2023,
       "relevance": "Reviews applications of LLMs in psychology research including as substitutes for human participants."
     },
     {
       "title": "PersonaLLM: Investigating the ability of GPT-3.5 to express personality traits and gender differences",
       "authors": ["Hang Jiang", "Xiajie Zhang", "Xubo Cao", "Jad Kabbara", "Derry Roy"],
       "year": 2023,
       "arxiv_id": "2305.02547",
       "relevance": "Evaluates LLMs' ability to embody assigned personality traits in user personas, directly relevant to persona-based prompting approaches."
     },
     {
       "title": "Evaluating large language models in generating synthetic HCI research data: A case study",
       "authors": ["Perttu Hämäläinen", "Mikke Tavast", "Anton Kunnari"],
       "year": 2023,
       "doi": "10.1145/3544548.3580688",
       "relevance": "Evaluates LLMs for generating synthetic qualitative data in HCI research, finding they can produce believable accounts."
     },
     {
       "title": "Using large language models to simulate multiple humans and replicate human subject studies",
       "authors": ["Gati V. Aher", "Rosa I. Arriaga", "Adam Tauman Kalai"],
       "year": 2023,
       "relevance": "Presents the Turing Experiment test for evaluating how AI can simulate groups of participants, demonstrating replication of classic experiments."
     },
     {
       "title": "Large language models as subpopulation representative models: A review",
       "authors": ["Gabriel Simmons", "Christopher Hare"],
       "year": 2023,
       "arxiv_id": "2310.17888",
       "relevance": "Reviews the use of LLMs as representative models for subpopulations, directly relevant to the mega-persona approach."
     },
     {
       "title": "A prompt pattern catalog to enhance prompt engineering with ChatGPT",
       "authors": ["Jules White", "Quchen Fu", "Sam Hays"],
       "year": 2023,
       "arxiv_id": "2302.11382",
       "relevance": "Catalogs prompt engineering patterns including persona-based prompting, foundational to the approaches discussed in this paper."
     },
     {
       "title": "Supporting qualitative analysis with large language models: Combining codebook with GPT-3 for deductive coding",
       "authors": ["Ziang Xiao", "Xingdi Yuan", "Q. Vera Liao"],
       "year": 2023,
       "relevance": "Explores LLMs for qualitative data analysis, a complementary use to the data generation approach proposed in this paper."
     },
     {
       "title": "Guinea pigbots",
       "authors": ["Matthew Hutson", "Andrew Mastin"],
       "year": 2023,
       "relevance": "Science journal article discussing current and potential uses of generative AI in qualitative research, including as substitutes for human subjects."
     }
   ]
 }